From a98c5f01b5d093d95c463da72f2e230c09818aa2 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Tue, 15 Sep 2026 11:40:12 +0800 Subject: [PATCH 01/32] test(contract): refresh client_api_baseline before execution-guarantees work T0.1: capture current HEAD client-facing surface so the post-refactor comparison (T8.1) shows only this spec's changes. No code changes. --- .../server/contract/client_api_baseline.json | 66 ++++--------------- 1 file changed, 11 insertions(+), 55 deletions(-) diff --git a/tests/server/contract/client_api_baseline.json b/tests/server/contract/client_api_baseline.json index 65f9db14..61f0c48a 100644 --- a/tests/server/contract/client_api_baseline.json +++ b/tests/server/contract/client_api_baseline.json @@ -1,48 +1,4 @@ { - "data_plane": { - "paths": { - "/twinkle/append": { - "POST": { - "operationId": "append_twinkle_append_post", - "parameters": [], - "responses": [ - "200", - "422" - ] - } - }, - "/twinkle/get": { - "POST": { - "operationId": "get_twinkle_get_post", - "parameters": [], - "responses": [ - "200", - "422" - ] - } - }, - "/twinkle/put": { - "POST": { - "operationId": "put_twinkle_put_post", - "parameters": [], - "responses": [ - "200", - "422" - ] - } - }, - "/twinkle/release": { - "POST": { - "operationId": "release_twinkle_release_post", - "parameters": [], - "responses": [ - "200", - "422" - ] - } - } - } - }, "gateway": { "paths": { "/asample": { @@ -580,7 +536,7 @@ "paths": { "/healthz": { "GET": { - "operationId": "healthz_healthz_get", + "operationId": "model_healthz_healthz_get", "parameters": [], "responses": [ "200" @@ -777,9 +733,9 @@ ] } }, - "/twinkle/forward_from_data_plane": { + "/twinkle/forward_backward": { "POST": { - "operationId": "forward_from_data_plane_twinkle_forward_from_data_plane_post", + "operationId": "forward_backward_twinkle_forward_backward_post", "parameters": [], "responses": [ "200", @@ -787,9 +743,9 @@ ] } }, - "/twinkle/forward_backward": { + "/twinkle/forward_backward_from_data_plane": { "POST": { - "operationId": "forward_backward_twinkle_forward_backward_post", + "operationId": "forward_backward_from_data_plane_twinkle_forward_backward_from_data_plane_post", "parameters": [], "responses": [ "200", @@ -797,9 +753,9 @@ ] } }, - "/twinkle/forward_backward_from_data_plane": { + "/twinkle/forward_from_data_plane": { "POST": { - "operationId": "forward_backward_from_data_plane_twinkle_forward_backward_from_data_plane_post", + "operationId": "forward_from_data_plane_twinkle_forward_from_data_plane_post", "parameters": [], "responses": [ "200", @@ -1074,9 +1030,9 @@ ] } }, - "/twinkle/sample_to_data_plane": { + "/twinkle/sample_stream": { "POST": { - "operationId": "sample_to_data_plane_twinkle_sample_to_data_plane_post", + "operationId": "sample_stream_twinkle_sample_stream_post", "parameters": [], "responses": [ "200", @@ -1084,9 +1040,9 @@ ] } }, - "/twinkle/sample_stream": { + "/twinkle/sample_to_data_plane": { "POST": { - "operationId": "sample_stream_twinkle_sample_stream_post", + "operationId": "sample_to_data_plane_twinkle_sample_to_data_plane_post", "parameters": [], "responses": [ "200", From c35978936b7e4cd4c0c21ef4bdc5aa756da147b8 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Tue, 15 Sep 2026 11:52:39 +0800 Subject: [PATCH 02/32] feat(infra): establish Sync_Dispatch_Path time bound (Phase 1, T1.1-T1.6) - execute_all_sync forwards timeout to ray.get (T1.1) - resolve effective ray.get timeout before choosing execute_method and flip priority to 'decorator wins, instance is fallback' at both dispatch sites; fix the 0-treated-as-falsy trap; bound __len__/__next__ bare ray.get (T1.2, T1.5) - decorator timeout=10 on ping; timeout=3600 on save/add_adapter_to_model/ resume_from_checkpoint/tinker_load/load_full_weights_from_path (T1.3) - TaskQueueConfig default execution_timeout 120->1800; effective_execution_timeout (0 -> 3600) as the single bound source; startup warning on 0 (T1.4) - infra unit tests, no GPU/Megatron/server deps (T1.6) --- src/twinkle/infra/__init__.py | 21 ++-- src/twinkle/infra/_ray/ray_helper.py | 15 ++- src/twinkle/model/megatron/megatron.py | 4 +- .../model/megatron/multi_lora_megatron.py | 6 +- .../model/transformers/transformers.py | 2 +- .../sampler/vllm_sampler/vllm_sampler.py | 2 +- .../server/model/backends/megatron_model.py | 4 +- .../server/model/backends/mock_model.py | 2 +- .../model/backends/transformers_model.py | 2 +- src/twinkle/server/utils/task_queue/config.py | 23 +++- src/twinkle/server/utils/task_queue/mixin.py | 4 + src/twinkle/server/utils/task_queue/worker.py | 9 +- tests/infra/test_ray_get_timeout.py | 116 ++++++++++++++++++ 13 files changed, 182 insertions(+), 28 deletions(-) create mode 100644 tests/infra/test_ray_get_timeout.py diff --git a/src/twinkle/infra/__init__.py b/src/twinkle/infra/__init__.py index 4758b341..3a80219b 100644 --- a/src/twinkle/infra/__init__.py +++ b/src/twinkle/infra/__init__.py @@ -996,7 +996,10 @@ def remote_function(dispatch: Union[Literal['slice', 'all', 'slice_dp', 'last_pp sync: If True, use synchronous execution (execute_all_sync) instead of async. Required for methods with NCCL collective operations (e.g., Megatron forward_backward). lazy_collect: Do lazy collect, this boolean value decides whether this function needs lazy collect. If setting to None, it will follow the global setting. - timeout: Timeout in seconds for ray.get() when collecting results. Instance attribute ``_ray_get_timeout`` overrides this. + timeout: Timeout in seconds for ray.get() when collecting results. The decorator's + explicitly declared value takes priority; the instance attribute ``_ray_get_timeout`` + is the fallback for methods that declare none (``timeout if timeout is not None + else instance``). enable_continous_work: Route each request to the least busy worker instead of slicing the batch over all of them, and return the results in the caller's order. This is what lets a batch smaller than the worker @@ -1044,7 +1047,13 @@ def wrapper(self, *args, **kwargs) -> T1: else: # This is the driver from ._ray import RayHelper - execute_method = RayHelper.execute_all_async if not sync else RayHelper.execute_all_sync + # Resolve the effective ray.get timeout before choosing execute_method: + # the decorator's explicit value wins, the instance attribute is the + # fallback. ``is not None`` (not ``or``) so that a decorator ``timeout=0`` + # is honored instead of falling back to unbounded waiting. + _rgt = timeout if timeout is not None else getattr(self, '_ray_get_timeout', None) + execute_method = RayHelper.execute_all_async if not sync else functools.partial( + RayHelper.execute_all_sync, timeout=_rgt) # Only classes whose workers run methods side by side need # this; elsewhere Ray already orders calls per actor. _concurrent_actor = bool(getattr(self, '_max_concurrency', None)) @@ -1060,8 +1069,7 @@ def wrapper(self, *args, **kwargs) -> T1: _batch_len = _cw_batch_len(args, kwargs) if _batch_len: return _run_continous_work(self, func.__name__, execute_method, _workers, args, kwargs, - _batch_len, - getattr(self, '_ray_get_timeout', None) or timeout) + _batch_len, _rgt) if RayHelper.has_ref(args, kwargs): # If has any object-ref, dispatch in worker, because we don't know the structure in the ref. # for example, dataloader returns any data list. @@ -1079,7 +1087,6 @@ def wrapper(self, *args, **kwargs) -> T1: # busy. _tracked_refs = _cw_register(self, func.__name__, result) if _concurrent_actor else [] # This is a result future, call it to get the actual result - _rgt = getattr(self, '_ray_get_timeout', None) or timeout result_func = RayHelper.do_get_and_collect_func( _collect_func, collect, result, device_mesh, timeout=_rgt) _local_lazy_collect = _lazy_collect @@ -1090,13 +1097,13 @@ def wrapper(self, *args, **kwargs) -> T1: if func.__name__ == '__len__': # Get the first result and ignore the `lazy_collect` import ray - return ray.get(result[0]) + return ray.get(result[0], timeout=_rgt) if func.__name__ == '__next__': import ray for _res in result: # raise when any worker raises StopIteration - stop = ray.get(_res[1]) + stop = ray.get(_res[1], timeout=_rgt) if stop: raise StopIteration() result = [_res[0] for _res in result] diff --git a/src/twinkle/infra/_ray/ray_helper.py b/src/twinkle/infra/_ray/ray_helper.py index ffd4e1a4..d3cbc3f5 100644 --- a/src/twinkle/infra/_ray/ray_helper.py +++ b/src/twinkle/infra/_ray/ray_helper.py @@ -137,10 +137,19 @@ def is_worker(): return RayHelper.ray_inited() and ray._private.worker.global_worker.mode == ray._private.worker.WORKER_MODE @staticmethod - def execute_all_sync(method_name: str, workers_and_args: List[Tuple[Any, List[Any], Dict[str, Any]]]): - """Execute method and return results.""" + def execute_all_sync(method_name: str, + workers_and_args: List[Tuple[Any, List[Any], Dict[str, Any]]], + timeout=None): + """Execute method and return results. + + ``timeout`` is passed to ``ray.get(list, timeout=)``, whose semantics are + the **total** wall-clock time to collect the whole list -- different from + ``LazyCollect``'s per-future timing (see ``do_get_and_collect_func``). + The two paths are each bounded on their own; the total-time semantics here + are strictly tighter. + """ import ray - return ray.get(RayHelper.execute_all_async(method_name, workers_and_args)) + return ray.get(RayHelper.execute_all_async(method_name, workers_and_args), timeout=timeout) @staticmethod def execute_all_async(method_name: str, workers_and_args: List[Tuple[Any, List[Any], Dict[str, Any]]]): diff --git a/src/twinkle/model/megatron/megatron.py b/src/twinkle/model/megatron/megatron.py index 851529c6..617cbdba 100644 --- a/src/twinkle/model/megatron/megatron.py +++ b/src/twinkle/model/megatron/megatron.py @@ -883,7 +883,7 @@ def clip_grad_and_step(self, max_grad_norm: float = 1.0, norm_type=2, **kwargs): self.zero_grad(**kwargs) self.lr_step(**kwargs) - @remote_function(dispatch='all', collect='first', sync=True) + @remote_function(dispatch='all', collect='first', sync=True, timeout=3600) def save(self, name: Optional[str] = None, output_dir: Optional[str] = None, @@ -1486,7 +1486,7 @@ def _patch_adapter(self, adapter_name: str, config_or_dir: Union[PeftConfig, str self._default_tokenizer = self.optimizer_group[adapter_name].template.processor self.active_group = adapter_name - @remote_function(dispatch='all', sync=True) + @remote_function(dispatch='all', sync=True, timeout=3600) def add_adapter_to_model( self, adapter_name: str, diff --git a/src/twinkle/model/megatron/multi_lora_megatron.py b/src/twinkle/model/megatron/multi_lora_megatron.py index ebda9150..8af62df4 100644 --- a/src/twinkle/model/megatron/multi_lora_megatron.py +++ b/src/twinkle/model/megatron/multi_lora_megatron.py @@ -291,7 +291,7 @@ def _load_multi_lora_optimizer(self, checkpoint_dir: str, adapter_name: str = '' if optimizer_config is not None and 'iteration' in state_dict: optimizer_config.cur_step = state_dict['iteration'] - @remote_function(dispatch='all', collect='first', sync=True) + @remote_function(dispatch='all', collect='first', sync=True, timeout=3600) def save(self, name, output_dir: Optional[str] = None, interval=1, **kwargs): adapter_name = kwargs.pop('adapter_name', None) self._check_adapter_valid(adapter_name) @@ -372,7 +372,7 @@ def load(self, name: str, output_dir: Optional[str] = None, **kwargs): if dist.is_initialized(): dist.barrier() - @remote_function(dispatch='all', collect='first', sync=True) + @remote_function(dispatch='all', collect='first', sync=True, timeout=3600) def resume_from_checkpoint(self, checkpoint_dir, *, resume_only_model=False, **kwargs): adapter_name = kwargs.pop('adapter_name', None) self._check_adapter_valid(adapter_name) @@ -403,7 +403,7 @@ def get_state_dict(self, **kwargs): self._check_adapter_valid(kwargs.get('adapter_name')) return self.multi_adapter.get_state_dict(**kwargs) - @remote_function(dispatch='all', sync=True) + @remote_function(dispatch='all', sync=True, timeout=3600) def add_adapter_to_model( self, adapter_name: str, diff --git a/src/twinkle/model/transformers/transformers.py b/src/twinkle/model/transformers/transformers.py index 0087cc7f..fcf856f8 100644 --- a/src/twinkle/model/transformers/transformers.py +++ b/src/twinkle/model/transformers/transformers.py @@ -1557,7 +1557,7 @@ def _restore_training_state(self, checkpoint_dir, *, adapter_name=''): return trainer_state - @remote_function(dispatch='all', collect='first', sync=True) + @remote_function(dispatch='all', collect='first', sync=True, timeout=3600) def resume_from_checkpoint(self, checkpoint_dir, *, resume_only_model=False, **kwargs): adapter_name = kwargs.get('adapter_name', '') diff --git a/src/twinkle/sampler/vllm_sampler/vllm_sampler.py b/src/twinkle/sampler/vllm_sampler/vllm_sampler.py index 7877ddb5..fe68b9d7 100644 --- a/src/twinkle/sampler/vllm_sampler/vllm_sampler.py +++ b/src/twinkle/sampler/vllm_sampler/vllm_sampler.py @@ -495,7 +495,7 @@ def unload_adapter_paths(self, adapter_paths: list[str]) -> None: """Unload policy snapshots from vLLM and clear cached requests.""" self._run_in_loop(self.engine.unload_lora_paths(adapter_paths)) - @remote_function(dispatch='all', collect='first', lazy_collect=False) + @remote_function(dispatch='all', collect='first', lazy_collect=False, timeout=3600) def load_full_weights_from_path(self, path: Optional[str] = None) -> int: """Load a full (non-LoRA) HF checkpoint into the engine's base model. diff --git a/src/twinkle/server/model/backends/megatron_model.py b/src/twinkle/server/model/backends/megatron_model.py index 56f9acc9..8c9018be 100644 --- a/src/twinkle/server/model/backends/megatron_model.py +++ b/src/twinkle/server/model/backends/megatron_model.py @@ -102,7 +102,7 @@ def tinker_calculate_metric(self, is_training, **kwargs): metric = super().calculate_metric(is_training, **kwargs) return clean_metrics(metric) - @remote_function(dispatch='all', sync=True) + @remote_function(dispatch='all', sync=True, timeout=3600) def tinker_load(self, checkpoint_dir: str, **kwargs): """Load checkpoint with token-based isolation support.""" token = kwargs.pop('token', None) @@ -135,7 +135,7 @@ def forward_backward(self, *, inputs: InputFeature | list[InputFeature] | Trajec output = super().forward_backward(inputs=inputs, **kwargs) return to_cpu_safe_output(output) - @remote_function(collect='first', lazy_collect=False) + @remote_function(collect='first', lazy_collect=False, timeout=10) def ping(self) -> bool: """Lightweight liveness probe for watchdog health checks.""" return True diff --git a/src/twinkle/server/model/backends/mock_model.py b/src/twinkle/server/model/backends/mock_model.py index 7bf9866c..b6a79bd3 100644 --- a/src/twinkle/server/model/backends/mock_model.py +++ b/src/twinkle/server/model/backends/mock_model.py @@ -240,7 +240,7 @@ def remove_adapter(self, adapter_name: str) -> None: def has_adapter(self, adapter_name: str) -> bool: return adapter_name in self._adapters - @remote_function(collect='first', lazy_collect=False) + @remote_function(collect='first', lazy_collect=False, timeout=10) def ping(self) -> bool: """Lightweight liveness probe for watchdog health checks.""" return True diff --git a/src/twinkle/server/model/backends/transformers_model.py b/src/twinkle/server/model/backends/transformers_model.py index 8dc503bb..3de2da59 100644 --- a/src/twinkle/server/model/backends/transformers_model.py +++ b/src/twinkle/server/model/backends/transformers_model.py @@ -114,7 +114,7 @@ def forward_backward(self, *, inputs: InputFeature | list[InputFeature] | Trajec output = super().forward_backward(inputs=inputs, **kwargs) return to_cpu_safe_output(output) - @remote_function(collect='first', lazy_collect=False) + @remote_function(collect='first', lazy_collect=False, timeout=10) def ping(self) -> bool: """Lightweight liveness probe for watchdog health checks.""" return True diff --git a/src/twinkle/server/utils/task_queue/config.py b/src/twinkle/server/utils/task_queue/config.py index 79d62095..fdeeaf28 100644 --- a/src/twinkle/server/utils/task_queue/config.py +++ b/src/twinkle/server/utils/task_queue/config.py @@ -11,6 +11,9 @@ from pydantic import BaseModel, ConfigDict, Field +# Substituted when execution_timeout is left at 0 ("no configured limit"). +_ZERO_EXECUTION_TIMEOUT_FALLBACK: float = 3600.0 + class TaskQueueConfig(BaseModel): """Configuration for task queue and rate limiting. @@ -20,7 +23,9 @@ class TaskQueueConfig(BaseModel): tps_limit: Maximum input tokens per second per user token. ``0`` disables. window_seconds: Sliding window for rate-limit calculations. Must be > 0. queue_timeout: Maximum time a task can wait in queue (seconds). - execution_timeout: Maximum time a task can execute (seconds). 0 means no limit. + execution_timeout: Maximum time a task can execute (seconds). ``0`` means "no + configured limit"; a finite bound of 3600s is substituted instead of + unbounded waiting (see ``effective_execution_timeout``). enabled: Whether rate limiting is enabled. token_cleanup_multiplier: Multiplier for token cleanup threshold. token_cleanup_interval: How often to run cleanup task (seconds). @@ -33,8 +38,22 @@ class TaskQueueConfig(BaseModel): tps_limit: float = Field(default=16000.0, ge=0) window_seconds: float = Field(default=1.0, gt=0) queue_timeout: float = Field(default=300.0, ge=0) - execution_timeout: float = Field(default=120.0, ge=0) + execution_timeout: float = Field(default=1800.0, ge=0) enabled: bool = True token_cleanup_multiplier: float = Field(default=10.0, ge=0) token_cleanup_interval: float = Field(default=60.0, ge=0) max_input_tokens: int = Field(default=16000, ge=1) + + @property + def effective_execution_timeout(self) -> float: + """The single source of the execution time bound. + + ``0`` is not rejected (that would fail existing deployments); it is read + as "no configured limit" and replaced by a finite fallback so the bound + is always positive. This value feeds both ``_ray_get_timeout`` and the + ComputeWorker's ``asyncio.wait_for`` -- there is no second, independently + configurable timeout. + """ + if self.execution_timeout > 0: + return self.execution_timeout + return _ZERO_EXECUTION_TIMEOUT_FALLBACK diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/utils/task_queue/mixin.py index dcdf805c..f00c5313 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/utils/task_queue/mixin.py @@ -58,6 +58,10 @@ def _init_task_queue(self, config: TaskQueueConfig | None = None, deployment_nam config. """ self._task_queue_config = config if config is not None else TaskQueueConfig() + if self._task_queue_config.execution_timeout == 0: + logger.warning( + '[TaskQueue] execution_timeout=0: a finite %.0fs bound has replaced unbounded waiting ' + '(deployment=%s).', self._task_queue_config.effective_execution_timeout, deployment_name or 'unknown') self._deployment_name = deployment_name self._task_metrics = get_task_metrics(deployment_name) if deployment_name else None diff --git a/src/twinkle/server/utils/task_queue/worker.py b/src/twinkle/server/utils/task_queue/worker.py index fdbb36d1..ce517b19 100644 --- a/src/twinkle/server/utils/task_queue/worker.py +++ b/src/twinkle/server/utils/task_queue/worker.py @@ -232,10 +232,9 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue f'type={task_type}, queue_key={queue_key}') with traced_operation(handler_span_name, attrs=handler_attrs): coro = task.coro_factory() - if self._config.execution_timeout > 0: - result = await asyncio.wait_for(coro, timeout=self._config.execution_timeout) - else: - result = await coro + # effective_execution_timeout is always positive (0 -> finite fallback), + # so wait_for is always in effect. + result = await asyncio.wait_for(coro, timeout=self._config.effective_execution_timeout) exec_time = time.monotonic() - exec_start logger.info(f'[ComputeWorker] Task {task.request_id} completed in {exec_time:.2f}s, type={task_type}') if task.persist_status: @@ -250,7 +249,7 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue except asyncio.TimeoutError: task_status = 'timeout' exec_time = time.monotonic() - exec_start - error = (f'Execution timeout exceeded: {self._config.execution_timeout}s, ' + error = (f'Execution timeout exceeded: {self._config.effective_execution_timeout}s, ' f'actual execution time: {exec_time:.2f}s') logger.error(f'[ComputeWorker] Task {task.request_id} TIMEOUT after {exec_time:.2f}s, ' f'type={task_type}, queue_key={queue_key}') diff --git a/tests/infra/test_ray_get_timeout.py b/tests/infra/test_ray_get_timeout.py new file mode 100644 index 00000000..f9949eac --- /dev/null +++ b/tests/infra/test_ray_get_timeout.py @@ -0,0 +1,116 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Unit tests for the Sync_Dispatch_Path time bound (spec T1.6 / R9#1). + +These exercise only ``twinkle.infra`` against a plain sleeping Ray actor. They +depend on neither GPU, Megatron, nor any ``src/twinkle/server/**`` component. +""" +from __future__ import annotations + +import pytest + +ray = pytest.importorskip('ray') + +import twinkle.infra as infra +from twinkle.infra._ray.ray_helper import RayHelper +from twinkle.infra import remote_function + + +@ray.remote +class _Sleeper: + """A plain Ray actor whose only method sleeps for a caller-supplied time.""" + + def slow(self, seconds: float): + import time + time.sleep(seconds) + return seconds + + +@pytest.fixture(scope='module', autouse=True) +def _ray_and_ray_mode(): + """Bring up Ray and put infra into 'ray' mode for the driver-side path.""" + ray.init(ignore_reinit_error=True, num_cpus=2, logging_level='ERROR') + prev_mode = infra._mode + infra._mode = 'ray' + try: + yield + finally: + infra._mode = prev_mode + + +def _make_driver(): + """A minimal stand-in for a remote_class handle: one actor, no concurrency.""" + driver = type('Driver', (), {})() + driver._actors = [_Sleeper.remote()] + driver._max_concurrency = None + return driver + + +def test_execute_all_sync_times_out(_ray_and_ray_mode): + """R9#1: execute_all_sync(timeout=) raises when the remote does not return in time.""" + actor = _Sleeper.remote() + workers_and_args = [(actor, [3.0], {})] + with pytest.raises(ray.exceptions.GetTimeoutError): + RayHelper.execute_all_sync('slow', workers_and_args, timeout=0.5) + + +def test_execute_all_sync_returns_within_timeout(_ray_and_ray_mode): + actor = _Sleeper.remote() + workers_and_args = [(actor, [0.1], {})] + assert RayHelper.execute_all_sync('slow', workers_and_args, timeout=10.0) == [0.1] + + +def test_decorator_timeout_takes_priority_over_instance(): + """A small decorator timeout wins over a large instance ``_ray_get_timeout``.""" + + def slow(self, seconds): # body runs in the worker, not here + return seconds + + wrapped = remote_function(dispatch='all', collect='first', sync=True, timeout=0.5)(slow) + driver = _make_driver() + driver._ray_get_timeout = 100.0 # would allow the call if it were consulted + with pytest.raises(ray.exceptions.GetTimeoutError): + wrapped(driver, 3.0) + + +def test_decorator_timeout_wins_when_larger_than_instance(): + """The decorator value wins even when it is the *larger* of the two. + + A large decorator timeout with a tiny instance value must NOT time out -- + proving the instance value is ignored when the decorator declares one. + """ + + def slow(self, seconds): + return seconds + + wrapped = remote_function(dispatch='all', collect='first', sync=True, timeout=100.0)(slow) + driver = _make_driver() + driver._ray_get_timeout = 0.3 # would time out if it were consulted + result = wrapped(driver, 1.0) + # sync collect may hand back a lazy-collect callable; resolving it must not time out. + assert (result() if callable(result) else result) == 1.0 + + +def test_instance_timeout_is_fallback_when_decorator_absent(): + """With no decorator timeout, the instance ``_ray_get_timeout`` applies.""" + + def slow(self, seconds): + return seconds + + wrapped = remote_function(dispatch='all', collect='first', sync=True)(slow) + driver = _make_driver() + driver._ray_get_timeout = 0.3 + with pytest.raises(ray.exceptions.GetTimeoutError): + wrapped(driver, 2.0) + + +def test_decorator_timeout_zero_is_not_treated_as_falsy(): + """timeout=0 means 'time out immediately', not 'fall back to unbounded'.""" + + def slow(self, seconds): + return seconds + + wrapped = remote_function(dispatch='all', collect='first', sync=True, timeout=0)(slow) + driver = _make_driver() + driver._ray_get_timeout = 100.0 # the old ``or`` bug would fall back here + with pytest.raises(ray.exceptions.GetTimeoutError): + wrapped(driver, 1.0) From 1b030ea1a1f3dc00d57b86dbc582100b668768d9 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Tue, 15 Sep 2026 12:02:47 +0800 Subject: [PATCH 03/32] feat(contract): ErrorPayload contract + type base classes (Phase 2, T2.1-T2.4, T6.1) - twinkle_client/types/base.py: StrictRequest/ResponseModel/DataModel + backend_only() helper/reader; naming rulings in module docstring (defined, not applied) (T6.1) - twinkle_client/types/errors.py: ErrorCategory + ErrorPayload(ResponseModel) (T2.1) - task_errors.py: task_error_payload builds ErrorPayload dict (request_id/error_code, traceback split+tail-trim, User carries no traceback); error_payload_from_stored backfills legacy two-field payloads without ValidationError (T2.2) - worker: single-line error summary + full traceback in traceback field; TimeoutError and Ray_Get_Timeout -> 504/Server, others -> 500/Server (T2.3) - tests for ErrorPayload + updated task_errors test (T2.4) --- src/twinkle/server/utils/task_errors.py | 64 ++++++++++++- src/twinkle/server/utils/task_queue/mixin.py | 9 +- src/twinkle/server/utils/task_queue/worker.py | 35 ++++++-- src/twinkle_client/types/base.py | 90 +++++++++++++++++++ src/twinkle_client/types/errors.py | 50 +++++++++++ tests/server/state/test_error_payload.py | 63 +++++++++++++ tests/server/utils/test_task_errors.py | 35 ++++++-- 7 files changed, 327 insertions(+), 19 deletions(-) create mode 100644 src/twinkle_client/types/base.py create mode 100644 src/twinkle_client/types/errors.py create mode 100644 tests/server/state/test_error_payload.py diff --git a/src/twinkle/server/utils/task_errors.py b/src/twinkle/server/utils/task_errors.py index 478745c0..b0fa46eb 100644 --- a/src/twinkle/server/utils/task_errors.py +++ b/src/twinkle/server/utils/task_errors.py @@ -1,5 +1,65 @@ # Copyright (c) ModelScope Contributors. All rights reserved. +"""Construction and backward-compatible reading of failure payloads. +``ErrorPayload`` is the single representation of a failure both on the wire and in +state (R5). This module owns the two entry points that produce/repair it. +""" +from __future__ import annotations -def task_error_payload(error: str) -> dict[str, str]: - return {'error': error, 'category': 'Server'} +from collections.abc import Mapping +from typing import Any, Optional + +from twinkle_client.types.errors import ErrorCategory, ErrorPayload + +_ERROR_MAX = 1024 +_TRACEBACK_MAX = 65536 +_TRUNCATION_MARKER = '...[traceback truncated, tail kept]...\n' + + +def _trim_traceback(text: str) -> str: + """Keep the tail of an over-long traceback (innermost frames are densest).""" + if len(text) <= _TRACEBACK_MAX: + return text + keep = _TRACEBACK_MAX - len(_TRUNCATION_MARKER) + return _TRUNCATION_MARKER + text[-keep:] + + +def task_error_payload( + error: str, + *, + request_id: str, + error_code: int = 500, + category: ErrorCategory = ErrorCategory.Server, + traceback_text: Optional[str] = None, +) -> dict[str, Any]: + """Build an ``ErrorPayload`` and return it as a JSON-safe dict for storage. + + Traceback splitting and length trimming happen here so over-long text is never + written to State_Backend. A ``User`` category carries no traceback (R5#6). + """ + tb: Optional[str] = None + if category != ErrorCategory.User and traceback_text: + tb = _trim_traceback(traceback_text) + payload = ErrorPayload( + error=error[:_ERROR_MAX], + category=category, + error_code=error_code, + request_id=request_id, + traceback=tb, + ) + return payload.model_dump(mode='json') + + +def error_payload_from_stored(stored: Any, *, request_id: str) -> ErrorPayload: + """Build an ``ErrorPayload`` from whatever is sitting in ``FutureRecord.result``. + + Records written before this spec have only ``{error, category}``. Missing + ``error_code`` / ``request_id`` / ``category`` are backfilled with ``500`` / + the caller-supplied value / ``Unknown`` so a rolling upgrade never raises + ``pydantic.ValidationError``. + """ + data = dict(stored) if isinstance(stored, Mapping) else {'error': str(stored)} + data.setdefault('category', ErrorCategory.Unknown) + data.setdefault('error_code', 500) + data.setdefault('request_id', request_id) + return ErrorPayload.model_validate(data) diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/utils/task_queue/mixin.py index f00c5313..7a5193fd 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/utils/task_queue/mixin.py @@ -359,8 +359,13 @@ async def _run() -> None: queue_state=QueueState.ACTIVE.value, ) logger.info(f'[TaskQueue] Background task {request_id} completed, type={task_type or "unknown"}') - except Exception: - error_payload = task_error_payload(traceback.format_exc()) + except Exception as exc: + error_payload = task_error_payload( + f'{type(exc).__name__}: {exc}', + request_id=request_id, + error_code=500, + traceback_text=traceback.format_exc(), + ) await self.state.store_future_status( request_id, TaskStatus.FAILED.value, diff --git a/src/twinkle/server/utils/task_queue/worker.py b/src/twinkle/server/utils/task_queue/worker.py index ce517b19..f02ce959 100644 --- a/src/twinkle/server/utils/task_queue/worker.py +++ b/src/twinkle/server/utils/task_queue/worker.py @@ -27,6 +27,13 @@ logger = get_logger() +# Ray_Get_Timeout is classified the same as asyncio.TimeoutError: 504/Server (R5#8). +try: + from ray.exceptions import GetTimeoutError as _RayGetTimeout + _TIMEOUT_EXCEPTIONS: tuple[type[BaseException], ...] = (asyncio.TimeoutError, _RayGetTimeout) +except Exception: # pragma: no cover - ray always present in server runtime + _TIMEOUT_EXCEPTIONS = (asyncio.TimeoutError, ) + class ComputeWorker: """Serial background worker that processes GPU compute tasks. @@ -141,14 +148,22 @@ async def _store_task_failed( error: str, queue_state: str, queue_state_reason: str | None = None, + *, + error_code: int = 500, + traceback_text: str | None = None, ) -> None: - """Store FAILED status with a standardised error payload.""" + """Store FAILED status with a standardised ``ErrorPayload``.""" if task.persist_status: await self._state.store_future_status( task.request_id, TaskStatus.FAILED.value, task.model_id, - result=task_error_payload(error), + result=task_error_payload( + error, + request_id=task.request_id, + error_code=error_code, + traceback_text=traceback_text, + ), queue_state=queue_state, queue_state_reason=queue_state_reason, ) @@ -246,21 +261,25 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue queue_state=QueueState.ACTIVE.value, ) self._complete_result(task, result) - except asyncio.TimeoutError: + except _TIMEOUT_EXCEPTIONS: task_status = 'timeout' exec_time = time.monotonic() - exec_start - error = (f'Execution timeout exceeded: {self._config.effective_execution_timeout}s, ' + error = (f'Backend call timed out (bound {self._config.effective_execution_timeout}s), ' f'actual execution time: {exec_time:.2f}s') logger.error(f'[ComputeWorker] Task {task.request_id} TIMEOUT after {exec_time:.2f}s, ' f'type={task_type}, queue_key={queue_key}') - await self._store_task_failed(task, error, QueueState.ACTIVE.value) - except Exception: + # asyncio.TimeoutError and Ray_Get_Timeout are 504/Server (R5#8). + await self._store_task_failed(task, error, QueueState.ACTIVE.value, error_code=504) + except Exception as exc: task_status = 'failed' exec_time = time.monotonic() - exec_start - error = traceback.format_exc() + # error is a single-line summary; the full traceback goes only to the + # traceback field, never into `error` (R5#7). + error = f'{type(exc).__name__}: {exc}' logger.error(f'[ComputeWorker] Task {task.request_id} FAILED after {exec_time:.2f}s, ' f'type={task_type}:\n{traceback.format_exc(limit=3)}') - await self._store_task_failed(task, error, QueueState.ACTIVE.value) + await self._store_task_failed( + task, error, QueueState.ACTIVE.value, error_code=500, traceback_text=traceback.format_exc()) finally: q.task_done() self._record_execution_time(task_type, exec_time) diff --git a/src/twinkle_client/types/base.py b/src/twinkle_client/types/base.py new file mode 100644 index 00000000..ff06b8f1 --- /dev/null +++ b/src/twinkle_client/types/base.py @@ -0,0 +1,90 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Shared pydantic base classes and the naming rulings for the wire contract. + +This module is a public contract carrier imported across packages (Twinkle_Server +reverse-imports ``twinkle_client.types``); it therefore intentionally carries **no** +underscore prefix. + +Naming rulings (authoritative for all three split specs; kept in code, not only in +the spec, so a later reader cannot merge these away): + +1. Schema_Module modules imported across packages do NOT use an underscore prefix. + ``base.py`` / ``errors.py`` / ``lifecycle.py`` / ``data.py`` are public-contract + carriers; an underscore means "package-private", and a cross-package import of a + private module is a violation. Modules used only inside Twinkle_Client (never + imported by Twinkle_Server) are exempt. +2. New twinkle-native request models do NOT reuse a class name already present in + ``tinker.types``. Known collision to avoid: ``ForwardBackwardRequest``. Two + handlers import ``types`` from twinkle_client and from tinker respectively; a + same-named model is distinguished only by the import alias and is easy to + misread in a review diff. +3. The field expressing a failure-semantic category is named ``error_code``, NOT + ``status_code`` -- an execution-time failure is delivered with HTTP 200, so the + value is systematically unequal to the response status code. +4. A closed value set on a wire field is declared as ``Literal`` / enum, never a + bare ``str`` (see ``QueueStateLiteral`` in ``errors.py``). + +These three base classes are DEFINED here but NOT applied to any existing model by +this spec: applying ``extra='forbid'`` would immediately reject an old client's +request, which would break the zero-wire-change guarantee. +""" +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict, Field +from pydantic.fields import FieldInfo +from typing import Any, Optional + + +class StrictRequest(BaseModel): + """Request bodies. Typos fail loudly.""" + + model_config = ConfigDict(frozen=True, extra='forbid') + + +class ResponseModel(BaseModel): + """Response bodies. An old client tolerates new server fields.""" + + model_config = ConfigDict(frozen=True, extra='ignore') + + +class DataModel(BaseModel): + """Data-plane models (InputFeature / Trajectory on the wire). + + Same ConfigDict as ResponseModel, different reason -- which is why this is a + separate class and not an alias. ResponseModel's ``ignore`` exists so an old + client tolerates new response fields. DataModel's ``ignore`` exists so a + user's Preprocessor / Template may leave harmless extra keys (the original + columns left by ``dataset.map``, say) without the request being rejected. + + Do NOT "fix" this to inherit StrictRequest. Doing so rejects those extra keys + and breaks a large number of existing datasets. + """ + + model_config = ConfigDict(frozen=True, extra='ignore') + + +# Key under which backend-applicability metadata is stored in a field's +# ``json_schema_extra``. A single constant, helper and reader -- kept here with the +# base classes rather than in a module of their own (no isolation benefit). +BACKEND_ONLY_KEY = 'twinkle_backend_only' + + +def backend_only(*backends: str, **field_kwargs: Any) -> FieldInfo: + """Mark a model field as applicable only to the given backend(s). + + Attaches the backend tuple to the field's ``json_schema_extra`` under + ``BACKEND_ONLY_KEY``; read it back with :func:`read_backend_only`. + """ + extra = dict(field_kwargs.pop('json_schema_extra', None) or {}) + extra[BACKEND_ONLY_KEY] = tuple(backends) + return Field(json_schema_extra=extra, **field_kwargs) + + +def read_backend_only(field_info: FieldInfo) -> Optional[tuple[str, ...]]: + """Return the backend tuple a field was tagged with, or ``None`` if untagged.""" + extra = getattr(field_info, 'json_schema_extra', None) + if isinstance(extra, dict): + value = extra.get(BACKEND_ONLY_KEY) + if value is not None: + return tuple(value) + return None diff --git a/src/twinkle_client/types/errors.py b/src/twinkle_client/types/errors.py new file mode 100644 index 00000000..50c38b73 --- /dev/null +++ b/src/twinkle_client/types/errors.py @@ -0,0 +1,50 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Structured failure payload -- the single representation of a failure. + +Twinkle <-> tinker exception mapping (verified, kept here so a future new exception +class can be lined up against its tinker counterpart): + +- tinker ``RequestFailedError`` (``tinker/_exceptions.py:176-196``; carries + ``message`` / ``request_id`` / ``category``) is the "the task completed in a failed + terminal state" exception. Its ``category`` uses the same three values as + :class:`ErrorCategory` (``Unknown`` / ``Server`` / ``User``). + +``twinkle_client/utils/patch_tinker.py`` shows two SDKs can coexist in one process, +so a semantically-equal but differently-named exception must be lookup-able. +""" +from __future__ import annotations + +from enum import StrEnum +from pydantic import Field +from typing import Any, Optional + +from .base import ResponseModel + + +class ErrorCategory(StrEnum): + """Error attribution. Matches tinker's ``RequestErrorCategory``.""" + + Unknown = 'Unknown' + Server = 'Server' + User = 'User' + + +class ErrorPayload(ResponseModel): + """The single representation of a failure, on the wire and in state. + + ``error_code``, not ``status_code``: once server-request-lifecycle lands, an + execution-time failure is delivered with HTTP 200, so this value is + *systematically* unequal to the response status code. Keeping the name + ``status_code`` would make every reader misparse it once. The 400-599 range is + kept to reuse HTTP's semantic space, not to align with response codes. + + Inherits ``ResponseModel`` (``extra='ignore'``), so a future added field does + not make an old client fail to parse it. + """ + + error: str = Field(max_length=1024) + category: ErrorCategory + error_code: int = Field(ge=400, le=599) + request_id: str + traceback: Optional[str] = Field(default=None, max_length=65536) + details: Optional[list[dict[str, Any]]] = None diff --git a/tests/server/state/test_error_payload.py b/tests/server/state/test_error_payload.py new file mode 100644 index 00000000..b5483445 --- /dev/null +++ b/tests/server/state/test_error_payload.py @@ -0,0 +1,63 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Tests for ErrorPayload construction, backfill, and tinker-SDK wire compat. + +Spec: T2.4 / R9#7 / R8#5. +""" +from __future__ import annotations + +from twinkle.server.utils.task_errors import error_payload_from_stored, task_error_payload +from twinkle_client.types.errors import ErrorCategory, ErrorPayload + + +def test_two_field_legacy_backfills_error_code_and_request_id(): + """A pre-spec {error, category} payload backfills to 500 + passed request_id.""" + stored = {'error': 'boom', 'category': 'Server'} + + payload = error_payload_from_stored(stored, request_id='req_42') + + assert isinstance(payload, ErrorPayload) + assert payload.error_code == 500 + assert payload.request_id == 'req_42' + assert payload.error == 'boom' + + +def test_overlong_traceback_is_trimmed_tail_kept_with_marker(): + long_tb = 'X' * 10 + ('line\n' * 40000) # well over 65536 chars + assert len(long_tb) > 65536 + + payload = task_error_payload( + 'RuntimeError: boom', request_id='req_1', error_code=500, traceback_text=long_tb) + + tb = payload['traceback'] + assert tb is not None + assert len(tb) <= 65536 + assert 'truncated' in tb # truncation marker present + assert tb.endswith('line\n') # tail preserved + + +def test_user_category_carries_no_traceback(): + payload = task_error_payload( + 'invalid field', request_id='req_2', error_code=422, + category=ErrorCategory.User, traceback_text='Traceback (most recent call last): ...') + + assert payload['category'] == ErrorCategory.User.value + assert payload['traceback'] is None + + +def test_tinker_sdk_parses_six_field_like_two_field(): + """R8#5: tinker's RequestFailedResponse ignores extra fields, so a six-field + payload parses equal to a two-field one on the declared fields. + + tinker's RequestErrorCategory values are lowercase ('server'), so the payloads + here use that value; the point under test is that the four extra fields are + ignored, not the category spelling.""" + from tinker.types import RequestFailedResponse + + two = {'error': 'boom', 'category': 'server'} + six = {**two, 'error_code': 504, 'request_id': 'req_9', 'traceback': None, 'details': None} + + parsed_six = RequestFailedResponse.model_validate(six) + parsed_two = RequestFailedResponse.model_validate(two) + + assert parsed_six.error == parsed_two.error + assert parsed_six.category == parsed_two.category diff --git a/tests/server/utils/test_task_errors.py b/tests/server/utils/test_task_errors.py index 0c498664..698ad5b7 100644 --- a/tests/server/utils/test_task_errors.py +++ b/tests/server/utils/test_task_errors.py @@ -1,10 +1,31 @@ -from twinkle.server.utils.task_errors import task_error_payload +from twinkle.server.utils.task_errors import error_payload_from_stored, task_error_payload +from twinkle_client.types.errors import ErrorCategory -def test_task_error_payload_keeps_lora_traceback(): - error = 'Traceback...\nRuntimeError: No lora available for tenant session-default. Max loras: 3\n' +def test_task_error_payload_builds_error_payload_dict(): + error = 'RuntimeError: No lora available for tenant session-default. Max loras: 3' - assert task_error_payload(error) == { - 'error': error, - 'category': 'Server', - } + payload = task_error_payload(error, request_id='req_1', error_code=500) + + assert payload['error'] == error + assert payload['category'] == ErrorCategory.Server.value + assert payload['error_code'] == 500 + assert payload['request_id'] == 'req_1' + assert payload['traceback'] is None + + +def test_task_error_payload_user_category_drops_traceback(): + payload = task_error_payload( + 'bad input', request_id='req_2', error_code=400, category=ErrorCategory.User, traceback_text='Traceback...') + + assert payload['category'] == ErrorCategory.User.value + assert payload['traceback'] is None + + +def test_error_payload_from_stored_backfills_two_field_legacy(): + stored = {'error': 'boom', 'category': 'Server'} + + payload = error_payload_from_stored(stored, request_id='req_3') + + assert payload.error_code == 500 + assert payload.request_id == 'req_3' From b6e0bebc1463a3465a081235effecc81fca6053d Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Tue, 15 Sep 2026 12:30:47 +0800 Subject: [PATCH 04/32] feat(contract): QueueStateLiteral + naming disambiguation guard (Phase 6, T6.2-T6.3) - QueueStateLiteral in types/errors.py, values sourced to match server QueueState; consistency test asserts equal value sets (T6.2) - T6.3 realized as a guard test (per user ruling): twinkle_client already shares 18 public names with tinker.types by design (tinker-compatible client), so the literal 'no intersection' cannot hold without renaming twinkle. Guard instead asserts no src/twinkle module binds a tinker and a twinkle_client type to the same local name (tinker must be aliased when both coexist). --- src/twinkle_client/types/errors.py | 7 +- tests/twinkle_client/test_types_contract.py | 78 +++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 tests/twinkle_client/test_types_contract.py diff --git a/src/twinkle_client/types/errors.py b/src/twinkle_client/types/errors.py index 50c38b73..61685b1e 100644 --- a/src/twinkle_client/types/errors.py +++ b/src/twinkle_client/types/errors.py @@ -16,10 +16,15 @@ class can be lined up against its tinker counterpart): from enum import StrEnum from pydantic import Field -from typing import Any, Optional +from typing import Any, Literal, Optional from .base import ResponseModel +# Closed value set, kept in sync with the server-side ``QueueState`` enum values +# (a consistency test asserts the two sets are equal). Wire fields carrying a queue +# state declare this alias, never a bare ``str`` (naming ruling 4). +QueueStateLiteral = Literal['active', 'paused_rate_limit', 'paused_capacity', 'unknown'] + class ErrorCategory(StrEnum): """Error attribution. Matches tinker's ``RequestErrorCategory``.""" diff --git a/tests/twinkle_client/test_types_contract.py b/tests/twinkle_client/test_types_contract.py new file mode 100644 index 00000000..877f2bf9 --- /dev/null +++ b/tests/twinkle_client/test_types_contract.py @@ -0,0 +1,78 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Contract-base consistency and naming-disambiguation tests. + +- T6.2 / R7#9: ``QueueStateLiteral`` value set equals the server ``QueueState`` enum. +- T6.3 / R7#7: naming disambiguation guard. + +Note on T6.3 scope (agreed deviation from the literal wording): ``twinkle_client`` is +a tinker-compatible client and already shares many public class names with +``tinker.types`` at HEAD (``ForwardRequest``, ``SampleRequest``, ``Checkpoint`` ...), +so a literal "the two modules' public export names are disjoint" assertion cannot +hold and twinkle's names are intentionally NOT renamed. Instead this guard enforces +the invariant the ruling actually targets: no ``src/twinkle/**`` module may bind a +tinker type and a twinkle_client type to the *same local name* (tinker must be +aliased when both coexist) -- which is what makes a review diff misread possible. +""" +from __future__ import annotations + +import ast +import pathlib +import typing + +import twinkle +from twinkle.server.utils.task_queue.types import QueueState +from twinkle_client.types.errors import QueueStateLiteral + +_TWINKLE_SRC = pathlib.Path(twinkle.__file__).resolve().parent + + +def test_queue_state_literal_matches_server_enum(): + literal_values = set(typing.get_args(QueueStateLiteral)) + enum_values = {state.value for state in QueueState} + assert literal_values == enum_values, ( + f'QueueStateLiteral {literal_values} != QueueState {enum_values}') + + +def _origin(module: str | None) -> str | None: + """Classify an import's source module as 'tinker', 'twinkle_client', or None.""" + if not module: + return None + if module == 'tinker' or module.startswith('tinker.'): + return 'tinker' + if module == 'twinkle_client' or module.startswith('twinkle_client.'): + return 'twinkle_client' + return None + + +def _binding_collisions(tree: ast.AST) -> set[str]: + """Return local names bound to BOTH a tinker and a twinkle_client import.""" + tinker_names: set[str] = set() + twinkle_names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + origin = _origin(node.module) + if origin is None: + continue + for alias in node.names: + bound = alias.asname or alias.name + (tinker_names if origin == 'tinker' else twinkle_names).add(bound) + elif isinstance(node, ast.Import): + for alias in node.names: + origin = _origin(alias.name) + if origin is None: + continue + bound = alias.asname or alias.name.split('.')[0] + (tinker_names if origin == 'tinker' else twinkle_names).add(bound) + return tinker_names & twinkle_names + + +def test_no_tinker_twinkle_same_name_binding(): + offenders: dict[str, set[str]] = {} + for path in _TWINKLE_SRC.rglob('*.py'): + tree = ast.parse(path.read_text(), filename=str(path)) + collisions = _binding_collisions(tree) + if collisions: + offenders[str(path.relative_to(_TWINKLE_SRC))] = collisions + assert not offenders, ( + 'tinker and twinkle_client types bound to the same local name (alias tinker ' + f'to disambiguate): {offenders}') From e46345bd8b5406ccb014a03e188db6110ff9efcc Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Tue, 15 Sep 2026 12:51:34 +0800 Subject: [PATCH 05/32] feat(server): Blocking_Call_Boundary + Admission_Gate (Phase 3, T3.1-T3.8) - call_backend: dedicated ThreadPoolExecutor (no max_workers=1) + per-replica opt-in Admission_Gate; gate released from the worker thread's finally so a wait_for-cancelled coroutine cannot free it while the call is still in flight; fast-fail BackendBusyError when the gate is held by a leaked call (T3.1) - ModelManagement enables the gate; SamplerManagement does not - worker maps BackendBusyError -> 503/Server - mechanical: 27 model/twinkle + 14 model/tinker + 8 sampler/twinkle + 4 sampler/tinker + 3 model/app direct backend calls -> await call_backend (T3.2-T3.5) - check_model_health async + admit=False ping; _cleanup_adapter via gate; /healthz awaits (T3.5, T3.6) - AST static check over src/twinkle/server/** + shared exemptions file (T3.7) - blocking-boundary integration tests (T3.8) --- src/twinkle/server/model/app.py | 19 +++- src/twinkle/server/model/tinker_handlers.py | 37 ++++--- src/twinkle/server/model/twinkle_handlers.py | 81 +++++++++----- src/twinkle/server/sampler/tinker_handlers.py | 9 +- .../server/sampler/twinkle_handlers.py | 17 +-- src/twinkle/server/utils/task_queue/mixin.py | 77 ++++++++++++- src/twinkle/server/utils/task_queue/types.py | 10 ++ src/twinkle/server/utils/task_queue/worker.py | 10 +- .../integration/test_blocking_boundary.py | 101 ++++++++++++++++++ tests/server/static/__init__.py | 1 + .../server/static/backend_call_exemptions.py | 23 ++++ .../static/test_no_direct_backend_call.py | 78 ++++++++++++++ 12 files changed, 401 insertions(+), 62 deletions(-) create mode 100644 tests/server/integration/test_blocking_boundary.py create mode 100644 tests/server/static/__init__.py create mode 100644 tests/server/static/backend_call_exemptions.py create mode 100644 tests/server/static/test_no_direct_backend_call.py diff --git a/src/twinkle/server/model/app.py b/src/twinkle/server/model/app.py index a56eb7dd..95f122c9 100644 --- a/src/twinkle/server/model/app.py +++ b/src/twinkle/server/model/app.py @@ -133,7 +133,9 @@ def __init__(self, self._replica_registered = False # Initialize mixins - self._init_task_queue(queue_config, deployment_name='Model') + # ModelManagement opts into the Admission_Gate (collective mis-pairing + + # queue-bypassing _cleanup_adapter); SamplerManagement does not. + self._init_task_queue(queue_config, deployment_name='Model', enable_admission_gate=True) self._init_adapter_manager(**(adapter_config or {})) # Note: countdown task is started lazily in _ensure_sticky() @@ -178,15 +180,19 @@ async def shutdown(self) -> None: pass await self.data_plane.close() - def check_model_health(self) -> dict: + async def check_model_health(self) -> dict: """Probe model actors liveness via a lightweight ping. Returns a dict with 'healthy' (bool) and 'detail' (str). If the model actors are dead (e.g. OOM/SIGSEGV), the ping call will raise RayActorError, signalling the watchdog to restart. + + The ping goes through the Blocking_Call_Boundary with ``admit=False`` so it + never blocks the event loop yet never queues behind the Admission_Gate -- + the moment a probe matters most is while a call is stuck holding the gate. """ try: - result = self.model.ping() + result = await self.call_backend(self.model.ping, admit=False) if result is True: return {'healthy': True, 'detail': 'model actors alive'} return {'healthy': False, 'detail': f'unexpected ping result: {result}'} @@ -199,9 +205,12 @@ async def _cleanup_adapter(self, adapter_name: str) -> None: if self.train_mode == 'full': # No PEFT adapter to remove; restore clean base weights so the # next tenant does not inherit this tenant's trained weights. - self.model.reload_initial_weights() + # Takes the Admission_Gate: this path is driven by the background + # countdown and never enters Task_Queue, so the gate is what keeps + # it from colliding with an in-flight training call. + await self.call_backend(self.model.reload_initial_weights) else: - self.model.remove_adapter(adapter_name) + await self.call_backend(self.model.remove_adapter, adapter_name) self.unregister_resource(adapter_name) await self.state.unload_model(adapter_name) diff --git a/src/twinkle/server/model/tinker_handlers.py b/src/twinkle/server/model/tinker_handlers.py index c3df70ef..5d95719b 100644 --- a/src/twinkle/server/model/tinker_handlers.py +++ b/src/twinkle/server/model/tinker_handlers.py @@ -69,18 +69,21 @@ async def _create_adapter(): template = get_template_for_model(self.base_model) if self.is_full_mode: self.register_resource(adapter_name, token, session_id=body.session_id) - self.model.set_template(template, adapter_name=model_adapter, model_id=self.base_model) - self.model.set_processor('InputProcessor', adapter_name=model_adapter) - self.model.set_optimizer('Adam', adapter_name=model_adapter) + await self.call_backend( + self.model.set_template, template, adapter_name=model_adapter, model_id=self.base_model) + await self.call_backend(self.model.set_processor, 'InputProcessor', adapter_name=model_adapter) + await self.call_backend(self.model.set_optimizer, 'Adam', adapter_name=model_adapter) self.set_resource_state(adapter_name, 'grad_ready', False) else: # TODO: Make LoraConfig more flexible lora_cfg = LoraConfig(r=body.lora_config.rank, target_modules='all-linear') self.register_resource(adapter_name, token, session_id=body.session_id) - self.model.add_adapter_to_model(adapter_name=adapter_name, config_or_dir=lora_cfg) - self.model.set_template(template, adapter_name=adapter_name, model_id=self.base_model) - self.model.set_processor('InputProcessor', adapter_name=adapter_name) - self.model.set_optimizer('Adam', adapter_name=adapter_name) + await self.call_backend( + self.model.add_adapter_to_model, adapter_name=adapter_name, config_or_dir=lora_cfg) + await self.call_backend( + self.model.set_template, template, adapter_name=adapter_name, model_id=self.base_model) + await self.call_backend(self.model.set_processor, 'InputProcessor', adapter_name=adapter_name) + await self.call_backend(self.model.set_optimizer, 'Adam', adapter_name=adapter_name) self.set_resource_state(adapter_name, 'grad_ready', False) training_run_manager = create_training_run_manager(token, client_type='tinker') training_run_manager.save(_model_id, body) @@ -147,7 +150,8 @@ async def _do_forward(): model_adapter = self.resolve_model_adapter_name(adapter_name) datum_list = body.forward_input.data loss_fn_config = body.forward_input.loss_fn_config or {} - output, loss = self.model.tinker_forward_only( + output, loss = await self.call_backend( + self.model.tinker_forward_only, inputs=datum_list, adapter_name=model_adapter, **loss_fn_config) return types.ForwardBackwardOutput( loss_fn_output_type='CrossEntropyLossReturn', @@ -190,7 +194,8 @@ async def _do_forward_backward(): datum_list = body.forward_backward_input.data loss_fn = body.forward_backward_input.loss_fn loss_fn_config = body.forward_backward_input.loss_fn_config or {} - output, loss = self.model.tinker_forward_backward( + output, loss = await self.call_backend( + self.model.tinker_forward_backward, inputs=datum_list, adapter_name=model_adapter, loss_fn=loss_fn, **loss_fn_config) output_type = ('ImportanceSamplingLossReturn' if loss_fn == 'importance_sampling' else 'CrossEntropyLossReturn') @@ -238,9 +243,10 @@ async def _do_optim(): if not self.get_resource_state(adapter_name, 'grad_ready', False): raise RuntimeError(f'No accumulated gradients for adapter={adapter_name}; ' 'call forward_backward before optim_step') - self.model.tinker_step(adam_params=body.adam_params, adapter_name=model_adapter) + await self.call_backend(self.model.tinker_step, adam_params=body.adam_params, adapter_name=model_adapter) self.set_resource_state(adapter_name, 'grad_ready', False) - metrics = self.model.tinker_calculate_metric(is_training=True, adapter_name=model_adapter) + metrics = await self.call_backend( + self.model.tinker_calculate_metric, is_training=True, adapter_name=model_adapter) return types.OptimStepResponse(metrics=metrics) except Exception: logger.error(traceback.format_exc()) @@ -267,7 +273,8 @@ async def _do_save(): checkpoint_manager = create_checkpoint_manager(token, client_type='tinker') checkpoint_name = checkpoint_manager.get_ckpt_name(body.path) save_dir = checkpoint_manager.get_save_dir(model_id=body.model_id, is_sampler=False) - self.model.save( + await self.call_backend( + self.model.save, name=checkpoint_name, output_dir=save_dir, adapter_name=model_adapter, save_optimizer=True) tinker_path = checkpoint_manager.save(body.model_id, name=checkpoint_name, is_sampler=False) return types.SaveWeightsResponse(path=tinker_path, type='save_weights') @@ -298,7 +305,8 @@ async def _do_save_for_sampler(): # Must save the checkpoint in the twinkle format before calling model.save() tinker_path = checkpoint_manager.save(body.model_id, name=checkpoint_name, is_sampler=True) logger.info(f'Saving weights to {save_dir}') - self.model.save( + await self.call_backend( + self.model.save, name='latest', output_dir=save_dir, adapter_name=self.resolve_model_adapter_name(adapter_name), @@ -341,7 +349,8 @@ async def _do_load(): assert self.model is not None, 'Model not loaded, please load model first' adapter_name = self.get_adapter_name(adapter_name=body.model_id) self.assert_resource_exists(adapter_name) - self.model.tinker_load( + await self.call_backend( + self.model.tinker_load, checkpoint_dir=body.path, load_optimizer=body.optimizer, adapter_name=self.resolve_model_adapter_name(adapter_name), diff --git a/src/twinkle/server/model/twinkle_handlers.py b/src/twinkle/server/model/twinkle_handlers.py index 582aa5c0..673a9c9a 100644 --- a/src/twinkle/server/model/twinkle_handlers.py +++ b/src/twinkle/server/model/twinkle_handlers.py @@ -71,7 +71,7 @@ async def model_healthz( self: ModelManagement = Depends(self_fn), ) -> dict: """Deep health probe: pings underlying model actors to verify liveness.""" - result = self.check_model_health() + result = await self.check_model_health() if not result['healthy']: from fastapi.responses import JSONResponse return JSONResponse(status_code=503, content=result) @@ -108,7 +108,8 @@ async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} inputs = _parse_inputs(body.inputs) - ret = self.model.forward( + ret = await self.call_backend( + self.model.forward, inputs=inputs, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) return {'result': ret} @@ -139,7 +140,8 @@ async def _task(): self.assert_resource_exists(adapter_name) raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) - ret = self.model.forward( + ret = await self.call_backend( + self.model.forward, inputs=_parse_inputs(raw_inputs), adapter_name=adapter_name, **kwargs, @@ -193,7 +195,8 @@ async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} inputs = _parse_inputs(body.inputs) - ret = self.model.forward_only( + ret = await self.call_backend( + self.model.forward_only, inputs=inputs, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) return {'result': ret} @@ -222,7 +225,7 @@ async def _task(): raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) inputs = _parse_inputs(raw_inputs) kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) - ret = self.model.forward_only(inputs=inputs, adapter_name=adapter_name, **kwargs) + ret = await self.call_backend(self.model.forward_only, inputs=inputs, adapter_name=adapter_name, **kwargs) if body.output_ref is not None: rows = select_output_rows( ret, @@ -257,7 +260,8 @@ async def calculate_loss( async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - ret = self.model.calculate_loss(adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + ret = await self.call_backend( + self.model.calculate_loss, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) return {'result': ret} return await run_task( @@ -271,7 +275,8 @@ async def backward(request: Request, body: types.AdapterRequest, self: ModelMana async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - self.model.backward(adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + await self.call_backend( + self.model.backward, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='backward')) @@ -299,7 +304,8 @@ async def _task(): for key in inputs: if isinstance(inputs[key], list) and isinstance(first_element(inputs[key]), (int, float)): inputs[key] = torch.tensor(inputs[key]) - ret = self.model.forward_backward( + ret = await self.call_backend( + self.model.forward_backward, inputs=all_inputs, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) return {'result': ret} @@ -330,7 +336,8 @@ async def _task(): self.assert_resource_exists(adapter_name) raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) - ret = self.model.forward_backward( + ret = await self.call_backend( + self.model.forward_backward, inputs=_parse_inputs(raw_inputs), adapter_name=adapter_name, **kwargs, @@ -361,7 +368,8 @@ async def clip_grad_norm( async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - ret = self.model.clip_grad_norm(adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + ret = await self.call_backend( + self.model.clip_grad_norm, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) return {'result': str(ret)} return await run_task( @@ -375,7 +383,8 @@ async def step(request: Request, body: types.AdapterRequest, self: ModelManageme async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - self.model.step(adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + await self.call_backend( + self.model.step, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='step')) @@ -387,7 +396,8 @@ async def zero_grad(request: Request, body: types.AdapterRequest, self: ModelMan async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - self.model.zero_grad(adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + await self.call_backend( + self.model.zero_grad, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='zero_grad')) @@ -399,7 +409,8 @@ async def lr_step(request: Request, body: types.AdapterRequest, self: ModelManag async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - self.model.lr_step(adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + await self.call_backend( + self.model.lr_step, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='lr_step')) @@ -415,7 +426,8 @@ async def clip_grad_and_step( async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - self.model.clip_grad_and_step( + await self.call_backend( + self.model.clip_grad_and_step, max_grad_norm=body.max_grad_norm, norm_type=body.norm_type, adapter_name=self.resolve_model_adapter_name(adapter_name), @@ -437,7 +449,8 @@ async def get_train_configs( async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - ret = self.model.get_train_configs( + ret = await self.call_backend( + self.model.get_train_configs, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) return {'result': ret} @@ -452,7 +465,8 @@ async def set_loss(request: Request, body: types.SetLossRequest, self: ModelMana async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - self.model.set_loss( + await self.call_backend( + self.model.set_loss, body.loss_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_loss')) @@ -469,7 +483,8 @@ async def set_optimizer( async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - self.model.set_optimizer( + await self.call_backend( + self.model.set_optimizer, body.optimizer_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) await run_task( @@ -487,7 +502,8 @@ async def set_lr_scheduler( async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - self.model.set_lr_scheduler( + await self.call_backend( + self.model.set_lr_scheduler, body.scheduler_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) await run_task( @@ -510,7 +526,8 @@ async def _task(): model_id=adapter_name, name=checkpoint_name, is_sampler=body.is_sampler) # For sampler weights the actual data is always written to 'latest/'. model_save_name = 'latest' if body.is_sampler else checkpoint_name - checkpoint_dir = self.model.save( + checkpoint_dir = await self.call_backend( + self.model.save, name=model_save_name, output_dir=save_dir, adapter_name=self.resolve_model_adapter_name(adapter_name), @@ -530,7 +547,8 @@ async def _task(): extra_kwargs = body.model_extra or {} checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') resolved = checkpoint_manager.resolve_load_path(body.name) - self.model.load( + await self.call_backend( + self.model.load, name=resolved.checkpoint_name, output_dir=resolved.checkpoint_dir, adapter_name=self.resolve_model_adapter_name(adapter_name), @@ -556,7 +574,8 @@ async def _task(): checkpoint_dir = ( Path(resolved.checkpoint_dir, resolved.checkpoint_name).as_posix() if resolved.checkpoint_dir else body.name) - ret = self.model.resume_from_checkpoint( + ret = await self.call_backend( + self.model.resume_from_checkpoint, checkpoint_dir, resume_only_model=body.resume_only_model, adapter_name=self.resolve_model_adapter_name(adapter_name), @@ -682,7 +701,7 @@ async def _task(): # No PEFT adapter to add; the default optimizer group is used. self.set_resource_state(adapter_name, 'grad_ready', False) else: - self.model.add_adapter_to_model(adapter_name, config, **extra_kwargs) + await self.call_backend(self.model.add_adapter_to_model, adapter_name, config, **extra_kwargs) except Exception: self.unregister_resource(adapter_name) await self.state.unload_model(adapter_name) @@ -706,7 +725,8 @@ async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} patch_cls = deserialize_object(body.patch_cls) - self.model.apply_patch( + await self.call_backend( + self.model.apply_patch, patch_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='apply_patch')) @@ -724,7 +744,8 @@ async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} metric_cls = deserialize_object(body.metric_cls) - self.model.add_metric( + await self.call_backend( + self.model.add_metric, metric_cls, is_training=body.is_training, adapter_name=self.resolve_model_adapter_name(adapter_name), @@ -744,7 +765,8 @@ async def set_template( async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - self.model.set_template( + await self.call_backend( + self.model.set_template, body.template_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_template')) @@ -761,7 +783,8 @@ async def set_processor( async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - self.model.set_processor( + await self.call_backend( + self.model.set_processor, body.processor_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) await run_task( @@ -779,7 +802,8 @@ async def calculate_metric( async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - ret = self.model.calculate_metric( + ret = await self.call_backend( + self.model.calculate_metric, is_training=body.is_training, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) @@ -800,7 +824,8 @@ async def get_state_dict( async def _task(): self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} - ret = self.model.get_state_dict(adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + ret = await self.call_backend( + self.model.get_state_dict, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) return {'result': ret} return await run_task( diff --git a/src/twinkle/server/sampler/tinker_handlers.py b/src/twinkle/server/sampler/tinker_handlers.py index de43b4b9..921e4fdc 100644 --- a/src/twinkle/server/sampler/tinker_handlers.py +++ b/src/twinkle/server/sampler/tinker_handlers.py @@ -52,9 +52,9 @@ async def _do_sample(): # Set template for sampler based on model type template = get_template_for_model(self.model_id) - self.sampler.set_template(template, model_id=self.model_id) + await self.call_backend(self.sampler.set_template, template, model_id=self.model_id) # Reset prefix cache for new weights - self.sampler.reset_prefix_cache() + await self.call_backend(self.sampler.reset_prefix_cache) # Get model_path from body or sampling session model_path = body.model_path @@ -96,9 +96,10 @@ async def _do_sample(): if os.path.exists(os.path.join(adapter_uri, 'adapter_config.json')): lora_path = adapter_uri else: - self.sampler.load_full_weights_from_path(adapter_uri) + await self.call_backend(self.sampler.load_full_weights_from_path, adapter_uri) - responses = self.sampler.sample( + responses = await self.call_backend( + self.sampler.sample, inputs=[prompt_inputs] * body.num_samples, sampling_params=sampling_params, adapter_path=lora_path, diff --git a/src/twinkle/server/sampler/twinkle_handlers.py b/src/twinkle/server/sampler/twinkle_handlers.py index 609c25f2..16af8e70 100644 --- a/src/twinkle/server/sampler/twinkle_handlers.py +++ b/src/twinkle/server/sampler/twinkle_handlers.py @@ -231,13 +231,13 @@ async def _task(): checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') _, resolved_uri = checkpoint_manager.parse_adapter_uri(body.adapter_uri) # Reset prefix cache only when new weights are loaded - self.sampler.reset_prefix_cache() + await self.call_backend(self.sampler.reset_prefix_cache) # LoRA adapter dir (has adapter_config.json) vs full-parameter # HF checkpoint. Full checkpoints replace the sampler base model. if resolved_uri and os.path.exists(os.path.join(resolved_uri, 'adapter_config.json')): adapter_path = resolved_uri elif resolved_uri: - self.sampler.load_full_weights_from_path(resolved_uri) + await self.call_backend(self.sampler.load_full_weights_from_path, resolved_uri) # Parse inputs inputs = body.inputs @@ -259,7 +259,8 @@ async def _task(): params = SamplingParams.from_dict(body.sampling_params) # Sample - responses = self.sampler.sample( + responses = await self.call_backend( + self.sampler.sample, inputs, params, adapter_name=full_adapter_name, @@ -379,7 +380,7 @@ async def set_template( """Set the chat template for encoding Trajectory inputs.""" extra_kwargs = body.model_extra or {} with traced_operation('sampler.set_template'): - self.sampler.set_template(body.template_cls, **extra_kwargs) + await self.call_backend(self.sampler.set_template, body.template_cls, **extra_kwargs) return types.SetTemplateResponse() @app.post('/twinkle/add_adapter_to_sampler', response_model=types.AddAdapterResponse) @@ -396,7 +397,7 @@ async def add_adapter_to_sampler( config = LoraConfig(**body.config) if isinstance(body.config, dict) else body.config with traced_operation('sampler.add_adapter_to_sampler', attrs={MODEL_ID: self.model_id}): - self.sampler.add_adapter_to_sampler(full_adapter_name, config) + await self.call_backend(self.sampler.add_adapter_to_sampler, full_adapter_name, config) return types.AddAdapterResponse(adapter_name=full_adapter_name) @@ -409,7 +410,7 @@ async def apply_patch( extra_kwargs = body.model_extra or {} patch_cls = deserialize_object(body.patch_cls) with traced_operation('sampler.apply_patch'): - self.sampler.apply_patch(patch_cls, **extra_kwargs) + await self.call_backend(self.sampler.apply_patch, patch_cls, **extra_kwargs) @app.post('/twinkle/sample_stream') async def sample_stream( @@ -437,11 +438,11 @@ async def sample_stream( from twinkle.server.checkpoint import create_checkpoint_manager checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') _, resolved_uri = checkpoint_manager.parse_adapter_uri(body.adapter_uri) - self.sampler.reset_prefix_cache() + await self.call_backend(self.sampler.reset_prefix_cache) if resolved_uri and os.path.exists(os.path.join(resolved_uri, 'adapter_config.json')): adapter_path = resolved_uri elif resolved_uri: - self.sampler.load_full_weights_from_path(resolved_uri) + await self.call_backend(self.sampler.load_full_weights_from_path, resolved_uri) inputs = body.inputs if isinstance(inputs, list): diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/utils/task_queue/mixin.py index 7a5193fd..10b01bdd 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/utils/task_queue/mixin.py @@ -8,10 +8,12 @@ from __future__ import annotations import asyncio +import functools import time import traceback import uuid from collections.abc import Callable, Coroutine +from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Any from twinkle.server.telemetry.middleware import get_task_metrics @@ -19,7 +21,7 @@ from twinkle.utils.logger import get_logger from .config import TaskQueueConfig from .rate_limiter import RateLimiter -from .types import QueuedTask, QueueState, TaskStatus +from .types import BackendBusyError, QueuedTask, QueueState, TaskStatus from .worker import ComputeWorker if TYPE_CHECKING: @@ -50,12 +52,23 @@ class TaskQueueMixin: state: ServerState - def _init_task_queue(self, config: TaskQueueConfig | None = None, deployment_name: str = '') -> None: + def _init_task_queue( + self, + config: TaskQueueConfig | None = None, + deployment_name: str = '', + *, + enable_admission_gate: bool = False, + ) -> None: """Initialise the task queue, rate limiter, and compute worker. ``config`` must be a typed :class:`TaskQueueConfig` (the launcher passes the instance straight through). ``None`` constructs a default config. + + ``enable_admission_gate`` turns on the per-replica Admission_Gate + (:meth:`call_backend`). ``ModelManagement`` enables it; ``SamplerManagement`` + does not (vllm sampler owns its own concurrency and the weight-update / + generation mutual exclusion is covered by infra ``_cw_barrier``). """ self._task_queue_config = config if config is not None else TaskQueueConfig() if self._task_queue_config.execution_timeout == 0: @@ -83,8 +96,65 @@ def _init_task_queue(self, config: TaskQueueConfig | None = None, deployment_nam deployment_name=deployment_name, ) + # Blocking_Call_Boundary: a dedicated thread pool that moves every backend + # call off the event loop. Deliberately NOT max_workers=1 -- a call that + # leaks past its wait_for timeout keeps its thread; capping at one worker + # would let one leak block the whole queue forever. + self._backend_executor = ThreadPoolExecutor(thread_name_prefix='twinkle-backend') + # per-replica Admission_Gate; opt-in per deployment. + self._backend_admission: asyncio.Semaphore | None = ( + asyncio.Semaphore(1) if enable_admission_gate else None) + self._event_loop: asyncio.AbstractEventLoop | None = None + async def call_backend(self, fn: Callable[..., Any], /, *args: Any, admit: bool = True, **kwargs: Any) -> Any: + """The only place a Blocking_Backend_Call leaves the event loop. + + Covers every backend call in this replica process -- not just handlers. + "Does it enter Task_Queue" is deliberately NOT the exemption test: what this + boundary protects is event-loop responsiveness, orthogonal to queueing. + ``check_model_health()``'s ping, ``_cleanup_adapter()``'s two calls (driven + by a countdown, never queued) and the non-queued sampler endpoints are + exactly the calls that test would have missed. + + The Admission_Gate it takes when ``admit=True`` guards against collective + mis-pairing (a task issued to the same actors while a timed-out call is + still in flight) and against queue-bypassing paths (``_cleanup_adapter``) -- + NOT GPU parallelism (Ray already serialises calls per actor). ``admit=False`` + skips the gate: it is for liveness probes only, because the moment a probe + matters most is while a call is stuck, and that is exactly when the gate is + held by the stuck thread; a probe touches no collective. The sampler side + does not enable the gate -- infra ``_cw_barrier`` already covers it. + + When the gate is held by a leaked (timed-out) call, a new admitting call + fails fast with :class:`BackendBusyError` instead of queueing behind it. + """ + loop = asyncio.get_running_loop() + sem = self._backend_admission if admit else None + if sem is None: + return await loop.run_in_executor(self._backend_executor, functools.partial(fn, *args, **kwargs)) + + # asyncio.Semaphore.acquire() does not yield when the gate is free, so this + # check-then-acquire is race-free for a Semaphore(1): if not locked here, + # acquire succeeds synchronously. + if sem.locked(): + raise BackendBusyError( + 'This replica is waiting for a timed-out backend call to exit; ' + 'refusing to queue behind it.') + await sem.acquire() + + def _work() -> Any: + try: + return fn(*args, **kwargs) + finally: + # Release only when the thread truly finishes. A wait_for timeout + # cancels the awaiting coroutine but NOT this thread; releasing on + # cancellation would free the gate while the backend call is still + # in flight (Property 4). + loop.call_soon_threadsafe(sem.release) + + return await loop.run_in_executor(self._backend_executor, _work) + @staticmethod def _queue_key(model_id: str | None, token: str | None) -> str: if model_id: @@ -422,4 +492,7 @@ async def shutdown_task_queue(self) -> None: """Gracefully shut down the compute queue and release resources.""" await self._rate_limiter.stop_cleanup_task() await self._compute_worker.stop() + # Do not wait on threads that may be leaked on a timed-out backend call. + if getattr(self, '_backend_executor', None) is not None: + self._backend_executor.shutdown(wait=False) logger.debug('[TaskQueue] Task queue shutdown complete') diff --git a/src/twinkle/server/utils/task_queue/types.py b/src/twinkle/server/utils/task_queue/types.py index daf8d2bb..a7283dd9 100644 --- a/src/twinkle/server/utils/task_queue/types.py +++ b/src/twinkle/server/utils/task_queue/types.py @@ -26,6 +26,16 @@ class TaskStatus(Enum): RATE_LIMITED = 'rate_limited' # Task rejected due to rate limiting +class BackendBusyError(RuntimeError): + """Raised when the per-replica Admission_Gate is held by a leaked backend call. + + A new backend call arriving while the gate is closed (its holder is a call that + already exceeded ``asyncio.wait_for`` but whose executor thread has not yet + returned) fails fast with this error instead of queueing behind it. The worker + maps it to ``ErrorPayload(category='Server', error_code=503)``. + """ + + class QueueState(Enum): """Queue state for tinker client compatibility. diff --git a/src/twinkle/server/utils/task_queue/worker.py b/src/twinkle/server/utils/task_queue/worker.py index f02ce959..954db4ee 100644 --- a/src/twinkle/server/utils/task_queue/worker.py +++ b/src/twinkle/server/utils/task_queue/worker.py @@ -19,7 +19,7 @@ from twinkle.server.utils.task_errors import task_error_payload from twinkle.utils.logger import get_logger from .config import TaskQueueConfig -from .types import QueuedTask, QueueState, TaskStatus +from .types import BackendBusyError, QueuedTask, QueueState, TaskStatus if TYPE_CHECKING: from twinkle.server.state import ServerState @@ -270,6 +270,14 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue f'type={task_type}, queue_key={queue_key}') # asyncio.TimeoutError and Ray_Get_Timeout are 504/Server (R5#8). await self._store_task_failed(task, error, QueueState.ACTIVE.value, error_code=504) + except BackendBusyError as exc: + task_status = 'failed' + exec_time = time.monotonic() - exec_start + error = str(exc) + logger.error(f'[ComputeWorker] Task {task.request_id} REFUSED (admission gate held) after ' + f'{exec_time:.2f}s, type={task_type}, queue_key={queue_key}') + # Gate held by a leaked timed-out call -> 503/Server (R2#4). + await self._store_task_failed(task, error, QueueState.ACTIVE.value, error_code=503) except Exception as exc: task_status = 'failed' exec_time = time.monotonic() - exec_start diff --git a/tests/server/integration/test_blocking_boundary.py b/tests/server/integration/test_blocking_boundary.py new file mode 100644 index 00000000..ddf0e5e8 --- /dev/null +++ b/tests/server/integration/test_blocking_boundary.py @@ -0,0 +1,101 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Blocking_Call_Boundary integration tests (T3.8 / R9#2 / Property 3-4). + +These exercise the real ``TaskQueueMixin.call_backend`` through a minimal harness +that sets only the two attributes it uses (a dedicated executor and the optional +Admission_Gate), constructed exactly as ``_init_task_queue`` does. The backend is a +deliberately slow plain callable -- no GPU, Megatron, or Ray involved. +""" +from __future__ import annotations + +import asyncio +import time +from concurrent.futures import ThreadPoolExecutor + +import pytest + +from twinkle.server.utils.task_queue.mixin import TaskQueueMixin +from twinkle.server.utils.task_queue.types import BackendBusyError + + +class _Harness(TaskQueueMixin): + """Minimal holder exposing the real call_backend with a chosen gate setting.""" + + def __init__(self, gate_enabled: bool) -> None: + self._backend_executor = ThreadPoolExecutor(thread_name_prefix='twinkle-backend') + self._backend_admission = asyncio.Semaphore(1) if gate_enabled else None + + def close(self) -> None: + self._backend_executor.shutdown(wait=False) + + +@pytest.mark.asyncio +async def test_healthz_style_probe_responsive_during_slow_backend(): + """Property 3: while a slow backend call is in flight, an admit=False probe + (as /healthz uses) returns well within 5 seconds.""" + h = _Harness(gate_enabled=True) + try: + slow = asyncio.create_task(h.call_backend(lambda: time.sleep(3.0))) + await asyncio.sleep(0.05) # let the slow call take the gate + a thread + + loop = asyncio.get_running_loop() + start = loop.time() + probe = await h.call_backend(lambda: 'pong', admit=False) # no gate, like the ping probe + elapsed = loop.time() - start + + assert probe == 'pong' + assert elapsed < 5.0 + await slow + finally: + h.close() + + +@pytest.mark.asyncio +async def test_gate_held_by_leaked_call_fast_fails_next_task(): + """Property 4 / R2#4: a call that outlives its wait_for keeps the gate; the next + admitting call fails fast with BackendBusyError instead of entering the backend.""" + h = _Harness(gate_enabled=True) + entered = {'count': 0} + + def slow(): + time.sleep(1.5) + + def would_enter_backend(): + entered['count'] += 1 + return 'should-not-run' + + try: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(h.call_backend(slow), timeout=0.3) + + # The leaked thread still holds the gate. + with pytest.raises(BackendBusyError): + await h.call_backend(would_enter_backend) + assert entered['count'] == 0 # never reached the backend + + # After the leaked thread truly finishes, the gate frees on its own. + await asyncio.sleep(1.6) + assert await h.call_backend(would_enter_backend) == 'should-not-run' + assert entered['count'] == 1 + finally: + h.close() + + +@pytest.mark.asyncio +async def test_sampler_without_gate_runs_two_calls_concurrently(): + """R9#2 case 3 / opt-in: with the gate disabled (SamplerManagement), two backend + calls are in flight at once rather than serialized.""" + h = _Harness(gate_enabled=False) + try: + loop = asyncio.get_running_loop() + start = loop.time() + results = await asyncio.gather( + h.call_backend(lambda: (time.sleep(1.0), 'a')[1]), + h.call_backend(lambda: (time.sleep(1.0), 'b')[1]), + ) + elapsed = loop.time() - start + + assert sorted(results) == ['a', 'b'] + assert elapsed < 1.8 # concurrent, not ~2.0s serialized + finally: + h.close() diff --git a/tests/server/static/__init__.py b/tests/server/static/__init__.py new file mode 100644 index 00000000..85b3e739 --- /dev/null +++ b/tests/server/static/__init__.py @@ -0,0 +1 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. diff --git a/tests/server/static/backend_call_exemptions.py b/tests/server/static/backend_call_exemptions.py new file mode 100644 index 00000000..1ba0674e --- /dev/null +++ b/tests/server/static/backend_call_exemptions.py @@ -0,0 +1,23 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Shared exemption list for the "no direct backend call" static checks. + +This file is the SINGLE source of allowed Blocking_Backend_Call bypasses. It is +consumed by this spec's check (``test_no_direct_backend_call.py``) and is intended +to be consumed unchanged by the ``server-request-lifecycle`` spec's equivalent +check -- there must be exactly one physical copy, not one per spec (R2#8). + +Each entry is ``(module_relpath, function_name)`` where ``module_relpath`` is +relative to ``src/twinkle/server`` and ``function_name`` is the innermost enclosing +function of the exempted call. + +The only allowed exemption is the ray ``Queue.get`` inside ``sample_stream``'s +``_stream_generator``: it bridges the sampler actor's process boundary and is bounded +by the dedicated double-timeout of R4#10-11 (T5.5), not by ``call_backend``. No +``remote_function`` call is exempt. +""" +from __future__ import annotations + +# (module_relpath under src/twinkle/server, innermost enclosing function name) +BACKEND_CALL_EXEMPTIONS: frozenset[tuple[str, str]] = frozenset({ + ('sampler/twinkle_handlers.py', '_stream_generator'), +}) diff --git a/tests/server/static/test_no_direct_backend_call.py b/tests/server/static/test_no_direct_backend_call.py new file mode 100644 index 00000000..4e2750ee --- /dev/null +++ b/tests/server/static/test_no_direct_backend_call.py @@ -0,0 +1,78 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Static check: no direct Blocking_Backend_Call in ``src/twinkle/server/**``. + +Spec: T3.7 / R2#7 / R2#8 / Property 2. + +Asserts that no module under ``src/twinkle/server`` invokes ``self.model.(...)`` +or ``self.sampler.(...)`` directly -- every such call must go through +``call_backend`` (the Blocking_Call_Boundary). The scan range is the directory +(not a file list), so a newly added handler file cannot silently escape it. + +Allowed bypasses are read from the shared ``backend_call_exemptions`` module, which +this spec and ``server-request-lifecycle`` are meant to consume unchanged. This spec +can only verify that *its* check reads that file; the two-spec binding holds once +the lifecycle check is wired to the same file. +""" +from __future__ import annotations + +import ast +import pathlib + +import twinkle +from tests.server.static.backend_call_exemptions import BACKEND_CALL_EXEMPTIONS + +_SERVER_ROOT = pathlib.Path(twinkle.__file__).resolve().parent / 'server' + + +def _is_self_backend_call(node: ast.Call) -> str | None: + """Return 'model'/'sampler' if node is a direct self.model/self.sampler.() call.""" + func = node.func + if not isinstance(func, ast.Attribute): + return None + owner = func.value # the object the method is called on + if (isinstance(owner, ast.Attribute) and owner.attr in ('model', 'sampler') + and isinstance(owner.value, ast.Name) and owner.value.id == 'self'): + return owner.attr + return None + + +class _Collector(ast.NodeVisitor): + + def __init__(self, relpath: str) -> None: + self.relpath = relpath + self.func_stack: list[str] = [] + self.offenders: list[tuple[str, str, int, str]] = [] + + def _visit_func(self, node: ast.AST) -> None: + self.func_stack.append(node.name) + self.generic_visit(node) + self.func_stack.pop() + + visit_FunctionDef = _visit_func + visit_AsyncFunctionDef = _visit_func + + def visit_Call(self, node: ast.Call) -> None: + owner = _is_self_backend_call(node) + if owner is not None: + enclosing = self.func_stack[-1] if self.func_stack else '' + if (self.relpath, enclosing) not in BACKEND_CALL_EXEMPTIONS: + self.offenders.append((self.relpath, enclosing, node.lineno, f'self.{owner}.{node.func.attr}')) + self.generic_visit(node) + + +def test_no_direct_backend_call_in_server(): + offenders: list[tuple[str, str, int, str]] = [] + for path in _SERVER_ROOT.rglob('*.py'): + relpath = str(path.relative_to(_SERVER_ROOT)) + collector = _Collector(relpath) + collector.visit(ast.parse(path.read_text(), filename=str(path))) + offenders.extend(collector.offenders) + + assert not offenders, ( + 'Direct backend calls must go through call_backend (or be listed in ' + f'backend_call_exemptions): {offenders}') + + +def test_exemptions_are_read_from_shared_file(): + # The shared file is the single source of allowed bypasses (R2#8). + assert ('sampler/twinkle_handlers.py', '_stream_generator') in BACKEND_CALL_EXEMPTIONS From 948daaab8bfb14c7b230f8e3fa43cc85391a0634 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Tue, 15 Sep 2026 13:01:50 +0800 Subject: [PATCH 06/32] feat(server): post-timeout actor recovery + two time bounds (Phase 4, T4.1-T4.3) - set _ray_get_timeout = effective execution timeout on model/sampler backends, effective for both sync and async dispatch (T4.1) - ComputeWorker fires an optional on_backend_timeout hook after a timeout; ModelManagement probes actor liveness (admit=False ping) and sets a health bit that /healthz reflects (503) and a successful probe auto-clears (T4.2) - worker skips a dequeued task whose record is already terminal (R3#8); document record-terminal (queue_timeout+T) vs resource-release (Collect_Width*T) bounds in EN+ZH Server docs (T4.3) --- .../Usage Guide/Server and Client/Server.md | 30 ++++++++ .../\346\234\215\345\212\241\347\253\257.md" | 15 ++++ src/twinkle/server/model/app.py | 27 ++++++-- src/twinkle/server/model/twinkle_handlers.py | 2 +- src/twinkle/server/sampler/app.py | 3 + src/twinkle/server/utils/task_queue/mixin.py | 6 ++ src/twinkle/server/utils/task_queue/worker.py | 27 ++++++++ .../server/integration/test_actor_recovery.py | 69 +++++++++++++++++++ 8 files changed, 174 insertions(+), 5 deletions(-) create mode 100644 tests/server/integration/test_actor_recovery.py diff --git a/docs/source_en/Usage Guide/Server and Client/Server.md b/docs/source_en/Usage Guide/Server and Client/Server.md index f31209b0..d57084f1 100644 --- a/docs/source_en/Usage Guide/Server and Client/Server.md +++ b/docs/source_en/Usage Guide/Server and Client/Server.md @@ -450,3 +450,33 @@ twinkle-server check-config -c server_config.yaml | `use_megatron: false` | `backend: transformers` | Additionally, this refactor introduces two new top-level fields — `telemetry` and `persistence` — which did not exist before. Add them as needed. + +## Execution time bounds + +Every backend call has a configuration-computable time bound. The single source of +the bound is the *effective execution timeout* `T`: it equals `execution_timeout` +from the task-queue config, or `3600s` when `execution_timeout` is set to `0` +("no configured limit"; a startup warning is logged). `T` drives both the +`ray.get` timeout on the backend and the `asyncio.wait_for` around each task — +there is no second, independently tunable timeout. The default `execution_timeout` +is `1800s`. + +Two distinct bounds follow, and they must not be collapsed into one number: + +| Bound | Expression | Meaning | +|-------|------------|---------| +| Record-terminal bound | `queue_timeout + T` | After this, a task's future record is guaranteed to be in a terminal state (`completed`/`failed`). Use it for alerting thresholds and client polling total-timeout. | +| Resource-release bound | `Collect_Width × T` | After this, the executor thread and the in-flight model-actor call for that task are guaranteed to have finished. Use it for capacity planning. | + +`Collect_Width = len(self._actors) = world_size = tp × pp × dp` — the number of +futures each `remote_function` collection waits on per call. Evidence: +`LazyCollect._get_result` iterates `self._futures`, which come from +`_get_workers(self._actors, execute)` (`infra/__init__.py`), covering every actor — +not just the data-parallel width. On a `tp=8` deployment the resource-release bound +is therefore `8 × T`, not `T`. + +The **difference** between the two bounds (`Collect_Width × T − (queue_timeout + T)`) +is the longest time the per-replica Admission_Gate can stay closed after a timeout: +the record is already terminal, but a leaked executor thread may still hold the gate +until its `ray.get` returns or raises. During that window newly arriving tasks fail +fast with a `Server`/503 error rather than queueing behind the stuck call. diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" index a4df7a2d..0794fb08 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" @@ -450,3 +450,18 @@ twinkle-server check-config -c server_config.yaml | `use_megatron: false` | `backend: transformers` | 此外本次重构新增了 `telemetry` 和 `persistence` 两个顶层字段（旧版本中不存在），可按需添加。 + +## 执行时间上界 + +每一次 backend 调用都存在一个可由配置计算的时间上界。上界的唯一来源是 **有效 execution timeout** `T`：它等于 task-queue 配置中的 `execution_timeout`；当 `execution_timeout` 配置为 `0`（表示「无配置上限」）时取 `3600` 秒，并在启动日志输出一条 warning。`T` 同时作用于 backend 的 `ray.get` 超时与每个任务外围的 `asyncio.wait_for`——不存在第二个可独立调节的超时值。`execution_timeout` 的默认值为 `1800` 秒。 + +由此派生出两个**不同**的上界，二者不得合成一个数： + +| 上界 | 表达式 | 含义 | +|------|--------|------| +| 记录终态上界 | `queue_timeout + T` | 超过它后，任务的 future 记录必处于终态（`completed`/`failed`）。用于设置告警阈值与客户端轮询总超时。 | +| 资源释放上界 | `Collect_Width × T` | 超过它后，该任务占用的 executor 线程与 model actor 在飞调用必已结束。用于容量规划。 | + +`Collect_Width = len(self._actors) = world_size = tp × pp × dp`——即每次 `remote_function` 结果收集所等待的 future 个数。证据：`LazyCollect._get_result` 遍历的 `self._futures` 来自 `_get_workers(self._actors, execute)`（`infra/__init__.py`），覆盖全部 actor，而非 data-parallel 宽度。因此在 `tp=8` 的部署上，资源释放上界是 `8 × T` 而非 `T`。 + +两个上界的**差值**（`Collect_Width × T − (queue_timeout + T)`）即 per-replica 准入闸门在一次超时后需保持关闭的最长时长：此时记录已是终态，但泄漏的 executor 线程可能仍持有闸门，直到其 `ray.get` 返回或抛出。在该窗口内新到达的任务会以 `Server`/503 错误快速失败，而不是排队等待卡住的调用。 diff --git a/src/twinkle/server/model/app.py b/src/twinkle/server/model/app.py index 95f122c9..cd85fb4b 100644 --- a/src/twinkle/server/model/app.py +++ b/src/twinkle/server/model/app.py @@ -131,11 +131,19 @@ def __init__(self, from twinkle.server.data_plane import DataPlaneProxy self.data_plane = DataPlaneProxy(data_plane_url) self._replica_registered = False + # Health status bit, set when a post-timeout probe finds the model actor + # unreachable; auto-cleared by the next successful probe (R3#3). + self._model_unhealthy = False # Initialize mixins # ModelManagement opts into the Admission_Gate (collective mis-pairing + # queue-bypassing _cleanup_adapter); SamplerManagement does not. - self._init_task_queue(queue_config, deployment_name='Model', enable_admission_gate=True) + self._init_task_queue( + queue_config, deployment_name='Model', enable_admission_gate=True, + on_backend_timeout=self._probe_after_timeout) + # Bound every ray.get on this backend by the effective execution timeout + # (applies to both sync=True and sync=False dispatch). T4.1. + self.model._ray_get_timeout = self._task_queue_config.effective_execution_timeout self._init_adapter_manager(**(adapter_config or {})) # Note: countdown task is started lazily in _ensure_sticky() @@ -183,9 +191,8 @@ async def shutdown(self) -> None: async def check_model_health(self) -> dict: """Probe model actors liveness via a lightweight ping. - Returns a dict with 'healthy' (bool) and 'detail' (str). - If the model actors are dead (e.g. OOM/SIGSEGV), the ping call - will raise RayActorError, signalling the watchdog to restart. + Returns a dict with 'healthy' (bool) and 'detail' (str). A successful probe + clears the unhealthy status bit; a failed probe sets it (R3#3). The ping goes through the Blocking_Call_Boundary with ``admit=False`` so it never blocks the event loop yet never queues behind the Admission_Gate -- @@ -194,11 +201,23 @@ async def check_model_health(self) -> dict: try: result = await self.call_backend(self.model.ping, admit=False) if result is True: + self._model_unhealthy = False return {'healthy': True, 'detail': 'model actors alive'} + self._model_unhealthy = True return {'healthy': False, 'detail': f'unexpected ping result: {result}'} except Exception as e: + self._model_unhealthy = True return {'healthy': False, 'detail': f'model actor unreachable: {e}'} + def mark_unhealthy(self) -> None: + """Flag the deployment unhealthy; /healthz returns 503 until a probe recovers it.""" + self._model_unhealthy = True + + async def _probe_after_timeout(self) -> None: + """Fired by ComputeWorker on a backend timeout: probe and log liveness (R3#2).""" + result = await self.check_model_health() + logger.warning('[Model] post-timeout liveness probe: %s', result) + async def _cleanup_adapter(self, adapter_name: str) -> None: if self.get_resource_info(adapter_name): self.clear_resource_state(adapter_name) diff --git a/src/twinkle/server/model/twinkle_handlers.py b/src/twinkle/server/model/twinkle_handlers.py index 673a9c9a..b780b279 100644 --- a/src/twinkle/server/model/twinkle_handlers.py +++ b/src/twinkle/server/model/twinkle_handlers.py @@ -72,7 +72,7 @@ async def model_healthz( ) -> dict: """Deep health probe: pings underlying model actors to verify liveness.""" result = await self.check_model_health() - if not result['healthy']: + if self._model_unhealthy or not result['healthy']: from fastapi.responses import JSONResponse return JSONResponse(status_code=503, content=result) return result diff --git a/src/twinkle/server/sampler/app.py b/src/twinkle/server/sampler/app.py index 8941a40b..892a66bd 100644 --- a/src/twinkle/server/sampler/app.py +++ b/src/twinkle/server/sampler/app.py @@ -129,6 +129,9 @@ def __init__(self, # Initialize task queue mixin self._init_task_queue(queue_config, deployment_name='Sampler') + # Bound every ray.get on this backend by the effective execution timeout + # (applies to both sync=True and sync=False dispatch). T4.1. + self.sampler._ray_get_timeout = self._task_queue_config.effective_execution_timeout async def shutdown(self) -> None: cancel_all = getattr(self.sampler, 'cancel_all_generations', None) diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/utils/task_queue/mixin.py index 10b01bdd..d3f31d2c 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/utils/task_queue/mixin.py @@ -58,6 +58,7 @@ def _init_task_queue( deployment_name: str = '', *, enable_admission_gate: bool = False, + on_backend_timeout: Callable[[], Coroutine[Any, Any, None]] | None = None, ) -> None: """Initialise the task queue, rate limiter, and compute worker. @@ -69,6 +70,10 @@ def _init_task_queue( (:meth:`call_backend`). ``ModelManagement`` enables it; ``SamplerManagement`` does not (vllm sampler owns its own concurrency and the weight-update / generation mutual exclusion is covered by infra ``_cw_barrier``). + + ``on_backend_timeout`` is an optional coroutine invoked once whenever a task + fails with a Ray_Get_Timeout / execution timeout, used by ModelManagement to + probe actor liveness (R3#2). """ self._task_queue_config = config if config is not None else TaskQueueConfig() if self._task_queue_config.execution_timeout == 0: @@ -94,6 +99,7 @@ def _init_task_queue( config=self._task_queue_config, task_metrics=self._task_metrics, deployment_name=deployment_name, + on_backend_timeout=on_backend_timeout, ) # Blocking_Call_Boundary: a dedicated thread pool that moves every backend diff --git a/src/twinkle/server/utils/task_queue/worker.py b/src/twinkle/server/utils/task_queue/worker.py index 954db4ee..1d951edb 100644 --- a/src/twinkle/server/utils/task_queue/worker.py +++ b/src/twinkle/server/utils/task_queue/worker.py @@ -53,11 +53,14 @@ def __init__( config: TaskQueueConfig, task_metrics: TaskMetrics | None, deployment_name: str, + on_backend_timeout: 'Callable[[], Any] | None' = None, ) -> None: self._state = state self._config = config self._task_metrics = task_metrics self._deployment_name = deployment_name + # Optional coroutine-returning callback fired on a backend timeout (R3#2). + self._on_backend_timeout = on_backend_timeout self.task_queues: dict[str, asyncio.Queue] = {} self.queue_order: Deque[str] = deque() @@ -270,6 +273,13 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue f'type={task_type}, queue_key={queue_key}') # asyncio.TimeoutError and Ray_Get_Timeout are 504/Server (R5#8). await self._store_task_failed(task, error, QueueState.ACTIVE.value, error_code=504) + # Probe actor liveness after a timeout so an operator learns the replica's + # state without waiting for a second request to also time out (R3#2). + if self._on_backend_timeout is not None: + try: + await self._on_backend_timeout() + except Exception: + logger.error(f'[ComputeWorker] backend-timeout probe failed:\n{traceback.format_exc(limit=3)}') except BackendBusyError as exc: task_status = 'failed' exec_time = time.monotonic() - exec_start @@ -322,12 +332,29 @@ async def _try_run_one(self) -> bool: await self._fail_timed_out_task(task, queue_wait, q) continue # try the next queue + # A record already in a Terminal_State (e.g. written 'failed' by the + # state-hygiene orphan handling) must not be executed again (R3#8). + if task.persist_status and await self._is_record_terminal(task.request_id): + logger.info(f'[ComputeWorker] Task {task.request_id} already terminal on dequeue; skipping.') + q.task_done() + continue + # Execute the task (serial: stops after the first execution) await self._execute_task(task, queue_key, q) return True return False + async def _is_record_terminal(self, request_id: str) -> bool: + """True if the future record already holds a Terminal_State.""" + try: + record = await self._state.get_future(request_id) + except Exception: + return False + if not record: + return False + return record.get('status') in (TaskStatus.COMPLETED.value, TaskStatus.FAILED.value) + # ------------------------------------------------------------------ # Main worker loop # ------------------------------------------------------------------ diff --git a/tests/server/integration/test_actor_recovery.py b/tests/server/integration/test_actor_recovery.py new file mode 100644 index 00000000..2ec4f807 --- /dev/null +++ b/tests/server/integration/test_actor_recovery.py @@ -0,0 +1,69 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Post-timeout liveness probe and health status bit (T4.2 / R3#2-3). + +Binds the real ``ModelManagement`` health methods onto a minimal harness with a +toggleable mock ``ping`` and a direct ``call_backend``. No GPU/Ray/full server. +""" +from __future__ import annotations + +import pytest + +from twinkle.server.model.app import ModelManagement + + +class _MockModel: + + def __init__(self) -> None: + self.alive = True + + def ping(self) -> bool: + if not self.alive: + raise RuntimeError('actor unreachable (simulated)') + return True + + +class _HealthHarness: + # Reuse the real implementations under test. + check_model_health = ModelManagement.check_model_health + mark_unhealthy = ModelManagement.mark_unhealthy + _probe_after_timeout = ModelManagement._probe_after_timeout + + def __init__(self, model: _MockModel) -> None: + self.model = model + self._model_unhealthy = False + + async def call_backend(self, fn, /, *args, admit: bool = True, **kwargs): + return fn(*args, **kwargs) + + +@pytest.mark.asyncio +async def test_timeout_probe_marks_unhealthy_then_recovers(): + model = _MockModel() + h = _HealthHarness(model) + + # Healthy at first. + result = await h.check_model_health() + assert result['healthy'] is True + assert h._model_unhealthy is False + + # A backend timeout fires the probe while the actor is unreachable. + model.alive = False + await h._probe_after_timeout() + assert h._model_unhealthy is True # /healthz would return 503 + + # Actor recovers; one successful probe clears the bit (no restart needed). + model.alive = True + result = await h.check_model_health() + assert result['healthy'] is True + assert h._model_unhealthy is False + + +@pytest.mark.asyncio +async def test_mark_unhealthy_is_cleared_by_successful_probe(): + h = _HealthHarness(_MockModel()) + h.mark_unhealthy() + assert h._model_unhealthy is True + + result = await h.check_model_health() + assert result['healthy'] is True + assert h._model_unhealthy is False From 742faf7a7ec87c13ccfcf645708604fd9ebaf496 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Tue, 15 Sep 2026 13:20:33 +0800 Subject: [PATCH 07/32] feat(state): future record survival + ownership hygiene (Phase 5, T5.1-T5.6) - FutureRecord.replica_id set at creation, never overwritten (T5.1) - ReplicaRegistry last_seen as a separate key (max_loras type unchanged); refreshed in _on_request_start; ModelManager.get_alive_replica_ids (T5.2) - cleanup_expired rewrite: never deletes a non-terminal record; orphans and over-absolute-ttl records are written failed; signature adds alive_replica_ids + absolute_ttl; ServerState.set_execution_bounds injects queue_timeout/T/ Collect_Width; age uses the stored-timestamp clock convention (T5.3) - do-not-regress guard extended to terminal->terminal (warn on different, silent drop on same) (T5.4) - sampler _stream_generator: 60s per-get + total-lifetime bound, ray Queue shutdown in finally, empty-actor structured error (T5.5) - state hygiene tests via Ray-free FileBackend (T5.6) --- src/twinkle/server/model/app.py | 10 ++ .../server/sampler/twinkle_handlers.py | 52 +++++++-- src/twinkle/server/state/future_manager.py | 101 +++++++++++++++--- src/twinkle/server/state/model_manager.py | 23 ++++ src/twinkle/server/state/models.py | 4 + src/twinkle/server/state/replica_registry.py | 37 +++++++ src/twinkle/server/state/server_state.py | 45 +++++++- src/twinkle/server/utils/task_queue/mixin.py | 3 + tests/server/state/test_future_lifecycle.py | 95 ++++++++++++++++ 9 files changed, 345 insertions(+), 25 deletions(-) create mode 100644 tests/server/state/test_future_lifecycle.py diff --git a/src/twinkle/server/model/app.py b/src/twinkle/server/model/app.py index cd85fb4b..6495dd7d 100644 --- a/src/twinkle/server/model/app.py +++ b/src/twinkle/server/model/app.py @@ -144,6 +144,15 @@ def __init__(self, # Bound every ray.get on this backend by the effective execution timeout # (applies to both sync=True and sync=False dispatch). T4.1. self.model._ray_get_timeout = self._task_queue_config.effective_execution_timeout + # Inject the execution bounds used by state hygiene to compute the absolute + # survival TTL for non-terminal future records (T5.3). Collect_Width = + # world_size = len(self._actors). + _actors = getattr(self.model, '_actors', None) + self.state.set_execution_bounds( + queue_timeout=self._task_queue_config.queue_timeout, + execution_timeout=self._task_queue_config.effective_execution_timeout, + collect_width=len(_actors) if _actors else 1, + ) self._init_adapter_manager(**(adapter_config or {})) # Note: countdown task is started lazily in _ensure_sticky() @@ -176,6 +185,7 @@ async def _ensure_sticky(self): async def _on_request_start(self, request: Request) -> str: await self._ensure_sticky() await self._ensure_replica_registered() + await self.state.touch_replica_last_seen(self.replica_id) await self._ensure_state_cleanup_started() token = get_token_from_request(request) return token diff --git a/src/twinkle/server/sampler/twinkle_handlers.py b/src/twinkle/server/sampler/twinkle_handlers.py index 16af8e70..19100adb 100644 --- a/src/twinkle/server/sampler/twinkle_handlers.py +++ b/src/twinkle/server/sampler/twinkle_handlers.py @@ -466,7 +466,14 @@ async def sample_stream( from .backends import STREAM_SENTINEL q = Queue(maxsize=128) - actor = self.sampler._actors[0] + actors = self.sampler._actors + if not actors: + # No available sampler actor -> structured error, not an IndexError (R4#12). + async def _no_actor_generator(): + yield json.dumps({'error': 'No available sampler actor', 'category': 'Server'}) + '\n' + + return StreamingResponse(_no_actor_generator(), media_type='application/x-ndjson') + actor = actors[0] actor.sample_stream_to_queue.remote( q, inputs_parsed, @@ -475,16 +482,41 @@ async def sample_stream( adapter_path=adapter_path, ) + # Two time bounds (R4#10-11): a per-get bound and a total-lifetime bound. + single_get_timeout = 60.0 + total_timeout = self._task_queue_config.effective_execution_timeout + async def _stream_generator(): loop = asyncio.get_event_loop() - while True: - item = await loop.run_in_executor(None, q.get) - if item == STREAM_SENTINEL: - break - if isinstance(item, Exception): - yield json.dumps({'error': str(item)}) + '\n' - break - delta, reason = item - yield json.dumps({'delta': delta, 'finish_reason': reason}) + '\n' + start = loop.time() + try: + while True: + remaining = total_timeout - (loop.time() - start) + if remaining <= 0: + yield json.dumps( + {'error': 'sample_stream exceeded the execution time bound', 'category': 'Server'}) + '\n' + break + try: + item = await asyncio.wait_for( + loop.run_in_executor(None, q.get), timeout=min(single_get_timeout, remaining)) + except asyncio.TimeoutError: + yield json.dumps( + {'error': 'sample_stream timed out waiting for the next token', 'category': 'Server'}) + '\n' + break + if item == STREAM_SENTINEL: + break + if isinstance(item, Exception): + yield json.dumps({'error': str(item)}) + '\n' + break + delta, reason = item + yield json.dumps({'delta': delta, 'finish_reason': reason}) + '\n' + finally: + # The run_in_executor(None, q.get) thread is NOT cancelled when + # wait_for times out; shutting the ray Queue down makes the blocked + # get() raise so the thread exits and the pool returns to baseline. + try: + q.shutdown(force=True) + except Exception: + pass return StreamingResponse(_stream_generator(), media_type='application/x-ndjson') diff --git a/src/twinkle/server/state/future_manager.py b/src/twinkle/server/state/future_manager.py index 5adf89cc..31f5cac3 100644 --- a/src/twinkle/server/state/future_manager.py +++ b/src/twinkle/server/state/future_manager.py @@ -5,10 +5,14 @@ from datetime import datetime from typing import Any +from twinkle.server.utils.task_errors import task_error_payload +from twinkle.utils.logger import get_logger from .backend.base import StateBackend from .base import BaseManager from .models import FutureRecord +logger = get_logger() + # Status sets used by the do-not-regress guard inside the atomic transform. _TERMINAL_STATUSES = frozenset({'completed', 'failed'}) _NON_TERMINAL_STATUSES = frozenset({'pending', 'queued', 'running'}) @@ -17,12 +21,14 @@ def _future_record_transform( existing: dict | None, *, + request_id: str, new_status: str, model_id: str | None, reason: str | None, result: Any, queue_state: str | None, queue_state_reason: str | None, + replica_id: str | None, now: str, ) -> dict | None: """Atomic transform body for :meth:`FutureManager.store_status`. @@ -30,12 +36,16 @@ def _future_record_transform( Module-level so it remains picklable when forwarded across the Ray actor boundary (closures and lambdas cannot be). - Drops the write entirely (returns ``None``) when ``new_status`` would - regress a terminal status — the StateBackend.update_atomic contract treats - a ``None`` return as "keep the current value", which is what stops stale - retries from clobbering a freshly committed terminal state. + A record already in a terminal state is never overwritten (returns ``None``, + which ``update_atomic`` treats as "keep current value"). A write of a + *different* terminal state is logged; a write of the *same* terminal state is + dropped silently (State_Backend idempotent retries produce these and they + indicate no defect). """ - if (existing is not None and existing.get('status') in _TERMINAL_STATUSES and new_status in _NON_TERMINAL_STATUSES): + existing_status = existing.get('status') if existing is not None else None + if existing_status in _TERMINAL_STATUSES: + if new_status != existing_status: + logger.warning('future %s already terminal as %r; refusing %r', request_id, existing_status, new_status) return None if existing is None: @@ -46,6 +56,7 @@ def _future_record_transform( result=result, queue_state=queue_state, queue_state_reason=queue_state_reason, + replica_id=replica_id, created_at=now, updated_at=now, ) @@ -55,6 +66,7 @@ def _future_record_transform( updated['status'] = new_status updated['model_id'] = model_id updated['updated_at'] = now + # replica_id is set at creation and is deliberately NOT overwritten here. if reason is not None: updated['reason'] = reason if result is not None: @@ -86,6 +98,7 @@ async def store_status( result: Any = None, queue_state: str | None = None, queue_state_reason: str | None = None, + replica_id: str | None = None, ) -> None: """Create or update a future record with the latest status. @@ -104,34 +117,94 @@ async def store_status( self._make_key(request_id), functools.partial( _future_record_transform, + request_id=request_id, new_status=status, model_id=model_id, reason=reason, result=result, queue_state=queue_state, queue_state_reason=queue_state_reason, + replica_id=replica_id, now=now, ), ) # ----- Cleanup ----- - async def cleanup_expired(self, cutoff_time: float, **kwargs) -> int: - """Remove futures whose last update is older than cutoff_time. + async def cleanup_expired( + self, + cutoff_time: float, + *, + alive_replica_ids: set[str] | None = None, + absolute_ttl: float | None = None, + ) -> int: + """Expire future records without ever deleting a non-terminal one. + + Processing matrix (design §5.2): + + | status | replica alive | over absolute_ttl | action | + |--------------|---------------|-------------------|-------------------| + | Terminal | — | ts < cutoff | delete | + | non-Terminal | yes | no | keep (untouched) | + | non-Terminal | yes | yes | write ``failed`` | + | non-Terminal | no | — | write ``failed`` | Args: - cutoff_time: Unix timestamp threshold. + cutoff_time: Unix timestamp; terminal records older than it are deleted. + alive_replica_ids: replicas currently considered alive. ``None`` disables + the orphan check (every non-terminal record is treated as owned). + absolute_ttl: seconds; a non-terminal record whose ``created_at`` is older + than this (regardless of ``updated_at``) is failed. ``None`` disables + the absolute-survival bound. Returns: - Number of futures removed. + Number of terminal records removed (records written ``failed`` are not + counted here; they are removed on a later pass once terminal). """ all_records = await self.get_all() - expired_ids = [] + # Use the same clock convention as the stored timestamps (_parse_timestamp on + # an ISO string) so the age computation is not skewed by _now_iso writing + # local time while _parse_timestamp reads naive ISO as UTC. + now = self._parse_timestamp(datetime.now().isoformat()) + expired_ids: list[str] = [] for request_id, record in all_records.items(): - timestamp_str = record.updated_at or record.created_at - timestamp = self._parse_timestamp(timestamp_str) - if timestamp < cutoff_time: - expired_ids.append(request_id) + if record.status in _TERMINAL_STATUSES: + timestamp = self._parse_timestamp(record.updated_at or record.created_at) + if timestamp < cutoff_time: + expired_ids.append(request_id) + continue + + # Non-terminal records are never deleted -- only ever written ``failed``. + # replica_id None (pre-upgrade) => ownership unknown => treated as alive. + replica_id = record.replica_id + replica_alive = (replica_id is None or alive_replica_ids is None or replica_id in alive_replica_ids) + if not replica_alive: + await self.store_status( + request_id, + 'failed', + record.model_id, + result=task_error_payload( + 'The replica that owned this task is no longer available.', + request_id=request_id, + error_code=503, + ), + replica_id=replica_id, + ) + continue + if absolute_ttl is not None: + created = self._parse_timestamp(record.created_at) + if (now - created) > absolute_ttl: + await self.store_status( + request_id, + 'failed', + record.model_id, + result=task_error_payload( + 'Task exceeded the absolute survival bound without reaching a terminal state.', + request_id=request_id, + error_code=500, + ), + replica_id=replica_id, + ) for request_id in expired_ids: await self.remove(request_id) diff --git a/src/twinkle/server/state/model_manager.py b/src/twinkle/server/state/model_manager.py index ae75621c..f4ccf034 100644 --- a/src/twinkle/server/state/model_manager.py +++ b/src/twinkle/server/state/model_manager.py @@ -11,6 +11,7 @@ from __future__ import annotations import functools +import time from .backend.base import StateBackend from .base import BaseManager @@ -101,6 +102,28 @@ async def unregister_replica(self, replica_id: str) -> None: await self.remove(model_id) await self._replicas.unregister(replica_id) + async def touch_replica_last_seen(self, replica_id: str) -> None: + """Refresh a replica's liveness timestamp (R4#6).""" + await self._replicas.touch_last_seen(replica_id) + + async def get_alive_replica_ids(self, liveness_threshold: float) -> set[str]: + """Return replicas considered alive (R4#7, R4#8). + + A replica is alive when it has a ``last_seen`` within ``liveness_threshold``, + OR when it has a ``max_loras`` entry but no ``last_seen`` yet (registered + before this spec / before its first request -- treated as alive so an + upgrade does not orphan in-flight tasks). + """ + registered = await self._replicas.get_all() + last_seen = await self._replicas.get_all_last_seen() + now = time.time() + alive: set[str] = set() + for rid in registered: + ls = last_seen.get(rid) + if ls is None or (now - ls) <= liveness_threshold: + alive.add(rid) + return alive + async def get_available_replica_ids(self, candidate_ids: list[str]) -> list[str]: """Return the subset of ``candidate_ids`` that still have capacity. diff --git a/src/twinkle/server/state/models.py b/src/twinkle/server/state/models.py index 71279b89..60970022 100644 --- a/src/twinkle/server/state/models.py +++ b/src/twinkle/server/state/models.py @@ -53,5 +53,9 @@ class FutureRecord(BaseModel): result: Any = None queue_state: str | None = None queue_state_reason: str | None = None + # Replica that produced the record; filled at creation and never overwritten by a + # later status update. ``None`` (pre-upgrade record) means "ownership unknown" and + # is only subject to the absolute survival bound (see FutureManager.cleanup_expired). + replica_id: str | None = None created_at: str = Field(default_factory=_now_iso) updated_at: str = Field(default_factory=_now_iso) diff --git a/src/twinkle/server/state/replica_registry.py b/src/twinkle/server/state/replica_registry.py index b2e11d13..d287c1d0 100644 --- a/src/twinkle/server/state/replica_registry.py +++ b/src/twinkle/server/state/replica_registry.py @@ -13,16 +13,23 @@ """ from __future__ import annotations +import time + from .backend.base import StateBackend REPLICA_PREFIX = 'replica::' _MAX_LORAS_SUFFIX = '::max_loras' +_LAST_SEEN_SUFFIX = '::last_seen' def _make_key(replica_id: str) -> str: return f'{REPLICA_PREFIX}{replica_id}{_MAX_LORAS_SUFFIX}' +def _last_seen_key(replica_id: str) -> str: + return f'{REPLICA_PREFIX}{replica_id}{_LAST_SEEN_SUFFIX}' + + def _replica_id_from_key(key: str) -> str | None: if not key.startswith(REPLICA_PREFIX) or not key.endswith(_MAX_LORAS_SUFFIX): return None @@ -42,6 +49,36 @@ async def register(self, replica_id: str, max_loras: int) -> None: async def unregister(self, replica_id: str) -> None: """Remove the capacity entry for ``replica_id`` (idempotent).""" await self._backend.delete(_make_key(replica_id)) + await self._backend.delete(_last_seen_key(replica_id)) + + async def touch_last_seen(self, replica_id: str) -> None: + """Refresh the replica's liveness timestamp (separate key from max_loras).""" + await self._backend.set(_last_seen_key(replica_id), time.time()) + + async def get_last_seen(self, replica_id: str) -> float | None: + """Return the replica's last-seen unix time, or ``None`` if never refreshed.""" + value = await self._backend.get(_last_seen_key(replica_id)) + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + async def get_all_last_seen(self) -> dict[str, float]: + """Return every replica's last-seen timestamp.""" + keys = await self._backend.keys(f'{REPLICA_PREFIX}*{_LAST_SEEN_SUFFIX}') + out: dict[str, float] = {} + for key in keys: + if not key.startswith(REPLICA_PREFIX) or not key.endswith(_LAST_SEEN_SUFFIX): + continue + rid = key[len(REPLICA_PREFIX):-len(_LAST_SEEN_SUFFIX)] + value = await self._backend.get(key) + try: + out[rid] = float(value) + except (TypeError, ValueError): + continue + return out async def get_max_loras(self, replica_id: str) -> int | None: """Return the declared capacity, or ``None`` if the replica is unknown.""" diff --git a/src/twinkle/server/state/server_state.py b/src/twinkle/server/state/server_state.py index 1fb548f5..e59b215d 100644 --- a/src/twinkle/server/state/server_state.py +++ b/src/twinkle/server/state/server_state.py @@ -39,6 +39,10 @@ LEASE_TTL = 30 # seconds — leader loses the lease after this without a renew LEASE_RENEW = 10 # seconds — must be < LEASE_TTL/2 so two missed renews still beat the TTL +# Integer multiple of (queue_timeout + resource-release bound) used as the absolute +# survival bound for a non-terminal record; the margin absorbs scheduling slack. +_ABSOLUTE_TTL_MULTIPLIER = 2 + def _renew_if_owner(current: str | None, *, owner: str) -> str | None: """``update_atomic`` transform: only re-write the lease if it is still mine.""" @@ -87,6 +91,13 @@ def __init__( self._cleanup_task: asyncio.Task | None = None self._cleanup_running = False + # Execution bounds injected by ModelManagement after _init_task_queue + # (see set_execution_bounds). Used to compute the absolute survival bound + # for non-terminal future records. All None => absolute bound disabled. + self._queue_timeout: float | None = None + self._effective_execution_timeout: float | None = None + self._collect_width: int | None = None + # Leader election + metrics-publish loop state. ``metrics_update_interval`` # is a typed parameter (a misspelled key now fails loudly rather than # being silently ignored); it controls how often the leader pushes counts @@ -289,6 +300,7 @@ async def store_future_status( result: Any = None, queue_state: str | None = None, queue_state_reason: str | None = None, + replica_id: str | None = None, ) -> None: """Store task status with optional result. @@ -317,6 +329,7 @@ async def store_future_status( result=result, queue_state=queue_state, queue_state_reason=queue_state_reason, + replica_id=replica_id, ) # ----- Configuration Management ----- @@ -370,7 +383,15 @@ async def cleanup_expired_resources(self) -> dict[str, int]: models_removed = await self._model_mgr.cleanup_expired(cutoff_time, expired_session_ids=expired_session_ids) samplings_removed = await self._sampling_mgr.cleanup_expired( cutoff_time, expired_session_ids=expired_session_ids) - futures_removed = await self._future_mgr.cleanup_expired(cutoff_time) + + # State hygiene for future records (design §5.2): protect non-terminal + # records owned by a live replica, fail orphans, and enforce the absolute + # survival bound. Alive set comes from ReplicaRegistry; absolute_ttl is + # computed only when the execution bounds were injected. + alive_replica_ids = await self._model_mgr.get_alive_replica_ids(self.expiration_timeout) + absolute_ttl = self._absolute_survival_ttl() + futures_removed = await self._future_mgr.cleanup_expired( + cutoff_time, alive_replica_ids=alive_replica_ids, absolute_ttl=absolute_ttl) return { 'sessions': sessions_removed, @@ -379,6 +400,28 @@ async def cleanup_expired_resources(self) -> dict[str, int]: 'futures': futures_removed, } + def set_execution_bounds(self, *, queue_timeout: float, execution_timeout: float, collect_width: int) -> None: + """Inject the execution bounds used to compute the absolute survival TTL. + + Called by ModelManagement after ``_init_task_queue``. ``ServerState`` is a + shared instance obtained via ``get_server_state()``, so these cannot be + constructor arguments; a setter is the injection path (design §5.2). + """ + self._queue_timeout = queue_timeout + self._effective_execution_timeout = execution_timeout + self._collect_width = collect_width + + def _absolute_survival_ttl(self) -> float | None: + """``k * (queue_timeout + Collect_Width * T)`` or ``None`` if not injected.""" + if (self._queue_timeout is None or self._effective_execution_timeout is None or self._collect_width is None): + return None + resource_release_bound = self._collect_width * self._effective_execution_timeout + return _ABSOLUTE_TTL_MULTIPLIER * (self._queue_timeout + resource_release_bound) + + async def touch_replica_last_seen(self, replica_id: str) -> None: + """Refresh a replica's liveness timestamp in the shared registry (R4#6).""" + await self._model_mgr.touch_replica_last_seen(replica_id) + async def _cleanup_loop(self) -> None: """Background task that periodically cleans up expired resources. diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/utils/task_queue/mixin.py index d3f31d2c..9887e259 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/utils/task_queue/mixin.py @@ -197,6 +197,7 @@ async def reject(error_msg: str, queue_state: str) -> dict[str, Any]: result=error_payload, queue_state=queue_state, queue_state_reason=error_msg, + replica_id=getattr(self, 'replica_id', None), ) return {'request_id': request_id, 'model_id': model_id} # Private marker consumed by schedule_task_and_wait(). It is not @@ -272,6 +273,7 @@ async def _schedule_task( TaskStatus.PENDING.value, model_id, queue_state=QueueState.ACTIVE.value, + replica_id=getattr(self, 'replica_id', None), ) queue_key = self._queue_key(model_id=model_id, token=token) @@ -422,6 +424,7 @@ async def schedule_background_task( TaskStatus.RUNNING.value, model_id, queue_state=QueueState.ACTIVE.value, + replica_id=getattr(self, 'replica_id', None), ) async def _run() -> None: diff --git a/tests/server/state/test_future_lifecycle.py b/tests/server/state/test_future_lifecycle.py new file mode 100644 index 00000000..bf687044 --- /dev/null +++ b/tests/server/state/test_future_lifecycle.py @@ -0,0 +1,95 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""State-hygiene tests for FutureManager cleanup and the do-not-regress guard. + +Spec: T5.6 / R9#5 / R9#6 / Property 6 / Property 7. Uses the Ray-free FileBackend. +""" +from __future__ import annotations + +from datetime import datetime +from unittest import mock + +import pytest + +from twinkle.server.state.future_manager import FutureManager + + +@pytest.fixture +def manager(tmp_path): + from twinkle.server.state.backend.file_backend import FileBackend + backend = FileBackend(str(tmp_path / 'state.json')) + return FutureManager(backend, expiration_timeout=300.0) + + +def _clock(manager) -> float: + """Now under the same convention the stored ISO timestamps use.""" + return manager._parse_timestamp(datetime.now().isoformat()) + + +async def _store(manager, request_id, status, *, replica_id=None): + await manager.store_status(request_id, status, model_id='m1', replica_id=replica_id) + + +@pytest.mark.asyncio +async def test_non_terminal_with_live_replica_is_kept(manager): + await _store(manager, 'r1', 'running', replica_id='replica-A') + removed = await manager.cleanup_expired( + cutoff_time=_clock(manager) + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) + assert removed == 0 + rec = await manager.get('r1') + assert rec is not None and rec.status == 'running' + + +@pytest.mark.asyncio +async def test_non_terminal_orphan_is_failed_not_deleted(manager): + await _store(manager, 'r2', 'running', replica_id='dead-replica') + await manager.cleanup_expired(cutoff_time=_clock(manager) + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) + rec = await manager.get('r2') + assert rec is not None # NOT deleted (Property 6) + assert rec.status == 'failed' + assert rec.result['category'] == 'Server' + + +@pytest.mark.asyncio +async def test_non_terminal_over_absolute_ttl_is_failed(manager): + await _store(manager, 'r3', 'running', replica_id='replica-A') + # absolute_ttl=0 makes any positive age exceed the bound. + await manager.cleanup_expired(cutoff_time=_clock(manager) + 10, alive_replica_ids={'replica-A'}, absolute_ttl=0.0) + rec = await manager.get('r3') + assert rec is not None and rec.status == 'failed' + + +@pytest.mark.asyncio +async def test_terminal_expired_is_deleted(manager): + await _store(manager, 'r4', 'completed', replica_id='replica-A') + removed = await manager.cleanup_expired( + cutoff_time=_clock(manager) + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) + assert removed == 1 + assert await manager.get('r4') is None + + +@pytest.mark.asyncio +async def test_terminal_to_terminal_different_is_refused_and_warns(manager): + await _store(manager, 'r5', 'failed', replica_id='replica-A') + with mock.patch('twinkle.server.state.future_manager.logger') as log: + await manager.store_status('r5', 'completed', model_id='m1') + rec = await manager.get('r5') + assert rec.status == 'failed' # not overwritten + assert log.warning.called + + +@pytest.mark.asyncio +async def test_terminal_to_terminal_same_is_dropped_without_warning(manager): + await _store(manager, 'r6', 'completed', replica_id='replica-A') + with mock.patch('twinkle.server.state.future_manager.logger') as log: + await manager.store_status('r6', 'completed', model_id='m1') + rec = await manager.get('r6') + assert rec.status == 'completed' + assert not log.warning.called + + +@pytest.mark.asyncio +async def test_replica_id_set_at_creation_not_overwritten(manager): + await _store(manager, 'r7', 'pending', replica_id='replica-A') + await manager.store_status('r7', 'running', model_id='m1', replica_id='replica-B') + rec = await manager.get('r7') + assert rec.replica_id == 'replica-A' # creation value preserved From f461fc373495675e2ec8b18af2825026c26d9811 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Tue, 15 Sep 2026 13:44:21 +0800 Subject: [PATCH 08/32] refactor: remove all silent degradation mechanisms (Phase 7, T7.1-T7.9) - delete TWINKLE_FAIL_FAST from the 4 cookbook server configs (T7.1) - delete Layer 1 (safe_loss/SafeLossWrapper/_zero_loss) + OptimizerGroup.__setattr__ auto-wrap hook + the test_micro_batch safe_loss import/case (T7.2) - delete Layer 2 (@nccl_safe decorator, _force_zero_backward, _iter_model_params) and its two decoration points + import in transformers_model (T7.3) - delete Layer 4: the forward_step_func post-processing try/except in megatron.py, exceptions now propagate; drop the _is_fail_fast import (T7.4) - delete Fail_Fast_Switch: _is_fail_fast + env_propagation NCCL_SAFE_ENV_KEYS/ build_nccl_safe_env_vars and its call (T7.5) - nccl_safe_megatron rewritten to unconditional rank-attributed log + re-raise, no tinker/forward_only params, no degraded return; module docstring records the lost coverage window (T7.6, T7.9) - rewrite both GPU-gated e2e tests to assert failed-terminal + subsequent-success and drop degradation symbols (T7.7) - unified static check asserting the 8 symbols are absent (T7.8) --- cookbook/client/async_rl/server_config.yaml | 8 - .../client/server/megatron/server_config.yaml | 3 - .../server/megatron/server_config_4b.yaml | 8 - .../server/transformer/server_config.yaml | 8 - src/twinkle/model/megatron/megatron.py | 97 ++-- src/twinkle/model/optimizer_group.py | 6 - .../server/launcher/env_propagation.py | 10 - .../server/model/backends/megatron_model.py | 6 +- .../model/backends/transformers_model.py | 3 - src/twinkle/utils/nccl_safe.py | 370 ++------------- tests/model/test_micro_batch.py | 12 - .../integration/test_nccl_safe_tinker_e2e.py | 432 ++---------------- .../integration/test_nccl_safe_twinkle_e2e.py | 331 ++------------ tests/server/static/test_no_degraded_path.py | 68 +++ 14 files changed, 238 insertions(+), 1124 deletions(-) create mode 100644 tests/server/static/test_no_degraded_path.py diff --git a/cookbook/client/async_rl/server_config.yaml b/cookbook/client/async_rl/server_config.yaml index 103ea523..6c0eb612 100644 --- a/cookbook/client/async_rl/server_config.yaml +++ b/cookbook/client/async_rl/server_config.yaml @@ -42,9 +42,6 @@ applications: target_ongoing_requests: 128 ray_actor_options: num_cpus: 0.1 - runtime_env: - env_vars: - TWINKLE_FAIL_FAST: "0" # TransferQueue-backed DataRef service. - name: data-plane @@ -95,7 +92,6 @@ applications: runtime_env: env_vars: TWINKLE_TRUST_REMOTE_CODE: "1" - TWINKLE_FAIL_FAST: "0" # A second GPU hosts vLLM and loads the same local base model. - name: sampler-Qwen3.5-4B @@ -133,7 +129,6 @@ applications: runtime_env: env_vars: TWINKLE_TRUST_REMOTE_CODE: "1" - TWINKLE_FAIL_FAST: "0" - name: processor route_prefix: /api/v1/processor @@ -155,6 +150,3 @@ applications: target_ongoing_requests: 128 ray_actor_options: num_cpus: 0.1 - runtime_env: - env_vars: - TWINKLE_FAIL_FAST: "0" diff --git a/cookbook/client/server/megatron/server_config.yaml b/cookbook/client/server/megatron/server_config.yaml index 24132a07..81d14e75 100644 --- a/cookbook/client/server/megatron/server_config.yaml +++ b/cookbook/client/server/megatron/server_config.yaml @@ -54,7 +54,6 @@ applications: env_vars: TWINKLE_TRUST_REMOTE_CODE: "0" TWINKLE_LONG_POLL_TIMEOUT: "120" - TWINKLE_FAIL_FAST: "0" # 3. Sampler Service - Runs inference / sampling using vLLM engine # Used for generating text from the model (e.g., evaluating LoRA results). @@ -98,7 +97,6 @@ applications: env_vars: TWINKLE_TRUST_REMOTE_CODE: "0" TWINKLE_LONG_POLL_TIMEOUT: "120" - TWINKLE_FAIL_FAST: "0" # 2. Model Service - Hosts the base model for training. # Config: PP=2 x DP=2 on 4 GPUs, ~27GB weights/GPU, comfortable for LoRA training @@ -139,4 +137,3 @@ applications: env_vars: TWINKLE_TRUST_REMOTE_CODE: "0" TWINKLE_LONG_POLL_TIMEOUT: "120" - TWINKLE_FAIL_FAST: "0" diff --git a/cookbook/client/server/megatron/server_config_4b.yaml b/cookbook/client/server/megatron/server_config_4b.yaml index 9bdbd5e7..7eed4699 100644 --- a/cookbook/client/server/megatron/server_config_4b.yaml +++ b/cookbook/client/server/megatron/server_config_4b.yaml @@ -31,9 +31,6 @@ applications: target_ongoing_requests: 128 # Target concurrent requests per replica ray_actor_options: num_cpus: 0.1 # CPU resources allocated to this actor - runtime_env: - env_vars: - TWINKLE_FAIL_FAST: "0" # 2. Model Service (commented out) - Would host the base model for training. # Uncomment and configure if you need a training model worker. @@ -71,7 +68,6 @@ applications: runtime_env: env_vars: TWINKLE_TRUST_REMOTE_CODE: "0" - TWINKLE_FAIL_FAST: "0" # 3. Sampler Service - Runs inference / sampling using vLLM engine # Used for generating text from the model (e.g., evaluating LoRA results). @@ -109,7 +105,6 @@ applications: runtime_env: env_vars: TWINKLE_TRUST_REMOTE_CODE: "0" - TWINKLE_FAIL_FAST: "0" # 4. Processor Service - name: processor @@ -132,6 +127,3 @@ applications: target_ongoing_requests: 128 ray_actor_options: num_cpus: 0.1 - runtime_env: - env_vars: - TWINKLE_FAIL_FAST: "0" diff --git a/cookbook/client/server/transformer/server_config.yaml b/cookbook/client/server/transformer/server_config.yaml index d3ddb2ad..b5d8497f 100644 --- a/cookbook/client/server/transformer/server_config.yaml +++ b/cookbook/client/server/transformer/server_config.yaml @@ -49,9 +49,6 @@ applications: target_ongoing_requests: 128 # Target concurrent requests per replica ray_actor_options: num_cpus: 0.1 # CPU resources allocated to this actor - runtime_env: - env_vars: - TWINKLE_FAIL_FAST: "0" # 2. Model Service - Hosts the base model for training. - name: models-Qwen3.5-4B @@ -85,7 +82,6 @@ applications: runtime_env: env_vars: TWINKLE_TRUST_REMOTE_CODE: "1" - TWINKLE_FAIL_FAST: "0" # 3. Sampler Service - Runs inference / sampling using vLLM engine # Used for generating text from the model (e.g., evaluating LoRA results). @@ -122,7 +118,6 @@ applications: runtime_env: env_vars: TWINKLE_TRUST_REMOTE_CODE: "1" - TWINKLE_FAIL_FAST: "0" # 4. Processor Service - name: processor @@ -145,6 +140,3 @@ applications: target_ongoing_requests: 128 ray_actor_options: num_cpus: 0.1 - runtime_env: - env_vars: - TWINKLE_FAIL_FAST: "0" diff --git a/src/twinkle/model/megatron/megatron.py b/src/twinkle/model/megatron/megatron.py index 617cbdba..89a05db7 100644 --- a/src/twinkle/model/megatron/megatron.py +++ b/src/twinkle/model/megatron/megatron.py @@ -36,7 +36,6 @@ from twinkle.processor import InputProcessor from twinkle.template import Template from twinkle.utils import construct_class, get_logger, selective_log_softmax -from twinkle.utils.nccl_safe import _is_fail_fast from ._mindspeed_runtime import ensure_mindspeed_adaptor_patched from .strategy import MegatronStrategy @@ -420,62 +419,48 @@ def forward_step_func(data_iterator, model): embeddings = None _loss_instance = loss_instance is_last_pp = mpu.is_pipeline_last_stage(False, unwrapped_model.vp_stage) - try: - if task == 'embedding': - # MegatronEmbeddingPatch already pooled output to [n_seqs, hidden] on last PP stage. - if is_last_pp: - embeddings = output_tensor - elif labels is not None and is_last_pp: - _loss_require_logps = getattr(_loss_instance, 'require_logps', True) - _loss_require_entropy = getattr(_loss_instance, 'require_entropy', False) - _packed = batch.get('packed_seq_params') - cu_seqlens_q = getattr(_packed, 'cu_seqlens_q', None) if _packed is not None else None - if _loss_require_logps: - loss_mask = (labels != -100).bool() - masked_labels = labels.clone() - masked_labels[~loss_mask] = 0 - output_tensor.div_(temperature) - if _loss_require_entropy: - logps, entropies = selective_log_softmax(output_tensor, masked_labels, return_entropy=True) - else: - logps = selective_log_softmax(output_tensor, masked_labels) - # Reconstruct full-length tensors from CP-split shards - logps = processor.postprocess_tensor_cp(logps, cu_seqlens=cu_seqlens_q) - if entropies is not None: - entropies = processor.postprocess_tensor_cp(entropies, cu_seqlens=cu_seqlens_q) - batch['labels'] = processor.postprocess_tensor_cp(labels, cu_seqlens=cu_seqlens_q) - if completion_mask is not None: - # Same index space as labels, so it needs the same CP reassembly. - batch['completion_mask'] = processor.postprocess_tensor_cp( - completion_mask, cu_seqlens=cu_seqlens_q) - if 'position_ids' in batch: - pos = batch['position_ids'] - if pos.dim() == 3: - pos = pos[0] # [2/3, 1, seq] → [1, seq] - batch['position_ids'] = processor.postprocess_tensor_cp(pos, cu_seqlens=cu_seqlens_q) - # Unpack packed sequences into per-sequence batch format - _outputs = {'logps': logps} + if task == 'embedding': + # MegatronEmbeddingPatch already pooled output to [n_seqs, hidden] on last PP stage. + if is_last_pp: + embeddings = output_tensor + elif labels is not None and is_last_pp: + _loss_require_logps = getattr(_loss_instance, 'require_logps', True) + _loss_require_entropy = getattr(_loss_instance, 'require_entropy', False) + _packed = batch.get('packed_seq_params') + cu_seqlens_q = getattr(_packed, 'cu_seqlens_q', None) if _packed is not None else None + if _loss_require_logps: + loss_mask = (labels != -100).bool() + masked_labels = labels.clone() + masked_labels[~loss_mask] = 0 + output_tensor.div_(temperature) + if _loss_require_entropy: + logps, entropies = selective_log_softmax(output_tensor, masked_labels, return_entropy=True) + else: + logps = selective_log_softmax(output_tensor, masked_labels) + # Reconstruct full-length tensors from CP-split shards + logps = processor.postprocess_tensor_cp(logps, cu_seqlens=cu_seqlens_q) if entropies is not None: - _outputs['entropies'] = entropies - if hasattr(_loss_instance, 'require_logits') and _loss_instance.require_logits: - _outputs['logits'] = output_tensor - batch, _outputs = processor.unpack_packed_sequences(batch, _outputs) - logps = _outputs['logps'] - entropies = _outputs.get('entropies', None) - unpacked_logits = _outputs.get('logits', None) - except Exception as e: - # Data processing error (e.g. unpack_packed_sequences dimension mismatch). - # Must catch here inside the scheduler to prevent exception escaping - # and breaking PP P2P communication → NCCL hang. - if _is_fail_fast(): - raise - logger.warning('[nccl_safe] forward_step_func data processing error: ' - '%s: %s', - type(e).__name__, e) - logps = None - unpacked_logits = None - entropies = None - embeddings = None + entropies = processor.postprocess_tensor_cp(entropies, cu_seqlens=cu_seqlens_q) + batch['labels'] = processor.postprocess_tensor_cp(labels, cu_seqlens=cu_seqlens_q) + if completion_mask is not None: + # Same index space as labels, so it needs the same CP reassembly. + batch['completion_mask'] = processor.postprocess_tensor_cp( + completion_mask, cu_seqlens=cu_seqlens_q) + if 'position_ids' in batch: + pos = batch['position_ids'] + if pos.dim() == 3: + pos = pos[0] # [2/3, 1, seq] → [1, seq] + batch['position_ids'] = processor.postprocess_tensor_cp(pos, cu_seqlens=cu_seqlens_q) + # Unpack packed sequences into per-sequence batch format + _outputs = {'logps': logps} + if entropies is not None: + _outputs['entropies'] = entropies + if hasattr(_loss_instance, 'require_logits') and _loss_instance.require_logits: + _outputs['logits'] = output_tensor + batch, _outputs = processor.unpack_packed_sequences(batch, _outputs) + logps = _outputs['logps'] + entropies = _outputs.get('entropies', None) + unpacked_logits = _outputs.get('logits', None) return output_tensor, partial( post_loss_function, inputs=batch, diff --git a/src/twinkle/model/optimizer_group.py b/src/twinkle/model/optimizer_group.py index f5177d67..150e694b 100644 --- a/src/twinkle/model/optimizer_group.py +++ b/src/twinkle/model/optimizer_group.py @@ -48,12 +48,6 @@ class BaseOptimizerGroup: _device_mesh: DeviceMesh = None _last_grad_norm: float = 0.0 - def __setattr__(self, name, value): - if name == 'loss_instance' and value is not None: - from twinkle.utils.nccl_safe import safe_loss - value = safe_loss(value) - super().__setattr__(name, value) - def do_grad_sync(self, gradient_accumulation_steps: Optional[int] = None) -> bool: if gradient_accumulation_steps is None: gradient_accumulation_steps = self.gradient_accumulation_steps diff --git a/src/twinkle/server/launcher/env_propagation.py b/src/twinkle/server/launcher/env_propagation.py index 3b0c9b95..4d4a7a47 100644 --- a/src/twinkle/server/launcher/env_propagation.py +++ b/src/twinkle/server/launcher/env_propagation.py @@ -21,10 +21,6 @@ 'TWINKLE_MODEL_ID_ALIASES', ) -# NCCL-safe env var keys: controls fault tolerance behavior in distributed -# training (safe_loss / @nccl_safe). Must reach model worker actors. -NCCL_SAFE_ENV_KEYS: tuple[str, ...] = ('TWINKLE_FAIL_FAST', ) - def build_telemetry_env_vars() -> dict[str, str]: """Collect telemetry env vars from ``os.environ`` for worker propagation.""" @@ -37,15 +33,9 @@ def build_persistence_env_vars() -> dict[str, str]: return {k: os.environ[k] for k in PERSISTENCE_ENV_KEYS if k in os.environ} -def build_nccl_safe_env_vars() -> dict[str, str]: - """Collect NCCL-safe env vars from ``os.environ`` for worker propagation.""" - return {k: os.environ[k] for k in NCCL_SAFE_ENV_KEYS if k in os.environ} - - def build_propagated_env_vars() -> dict[str, str]: """Aggregate all env vars that must reach Ray worker processes.""" merged: dict[str, str] = {} merged.update(build_telemetry_env_vars()) merged.update(build_persistence_env_vars()) - merged.update(build_nccl_safe_env_vars()) return merged diff --git a/src/twinkle/server/model/backends/megatron_model.py b/src/twinkle/server/model/backends/megatron_model.py index 8c9018be..fb3da3f2 100644 --- a/src/twinkle/server/model/backends/megatron_model.py +++ b/src/twinkle/server/model/backends/megatron_model.py @@ -33,7 +33,7 @@ class in the MRO. For full-parameter training the ``adapter_name`` is the """ @remote_function(dispatch='slice_dp', collect=collect_forward_backward_results, sync=True) - @nccl_safe_megatron(tinker=True) + @nccl_safe_megatron def tinker_forward_backward(self, *, inputs: list[types.Datum], adapter_name: str, loss_fn: str, **kwargs): """Combined forward and backward pass.""" self._tinker_setup_loss(loss_fn, inputs, adapter_name, kwargs) @@ -54,7 +54,7 @@ def tinker_forward_backward(self, *, inputs: list[types.Datum], adapter_name: st return [results, loss] @remote_function(dispatch='slice_dp', collect=collect_forward_backward_results) - @nccl_safe_megatron(tinker=True) + @nccl_safe_megatron def tinker_forward_only(self, *, inputs: list[types.Datum], adapter_name: str = None, **kwargs): """Forward pass without gradient computation.""" template = self.get_template(adapter_name) @@ -121,7 +121,7 @@ def tinker_load(self, checkpoint_dir: str, **kwargs): # ------------------------------------------------------------------ @remote_function(dispatch='slice_dp', collect=collect_tensor_dict) - @nccl_safe_megatron(forward_only=True) + @nccl_safe_megatron def forward_only(self, *, inputs: InputFeature | list[InputFeature] | Trajectory | list[Trajectory], **kwargs): """Forward-only for twinkle-native clients (InputFeature/Trajectory I/O).""" output = super().forward_only(inputs=inputs, **kwargs) diff --git a/src/twinkle/server/model/backends/transformers_model.py b/src/twinkle/server/model/backends/transformers_model.py index 3de2da59..a2bcba83 100644 --- a/src/twinkle/server/model/backends/transformers_model.py +++ b/src/twinkle/server/model/backends/transformers_model.py @@ -23,7 +23,6 @@ from twinkle.server.common.datum import datum_to_input_feature, extract_rl_features_for_loss from twinkle.server.model.backends.common import (TwinkleCompatModelBase, clean_metrics, collect_forward_backward_results, to_cpu_safe_output) -from twinkle.utils.nccl_safe import nccl_safe class _TransformersTinkerCompatMixin(TwinkleCompatModelBase): @@ -48,7 +47,6 @@ def tinker_forward_only(self, *, inputs: list[types.Datum], adapter_name: str = return [results, 0.0] @remote_function(dispatch='slice_dp', collect=collect_forward_backward_results) - @nccl_safe(tinker=True) def tinker_forward_backward(self, *, inputs: list[types.Datum], adapter_name: str, loss_fn: str, **kwargs): self._tinker_setup_loss(loss_fn, inputs, adapter_name, kwargs) template = self.get_template(adapter_name) @@ -107,7 +105,6 @@ def forward_only(self, *, inputs: InputFeature | list[InputFeature] | Trajectory return to_cpu_safe_output(output) @remote_function(dispatch='slice_dp', collect=collect_tensor_dict) - @nccl_safe def forward_backward(self, *, inputs: InputFeature | list[InputFeature] | Trajectory | list[Trajectory], **kwargs): """Forward+backward for twinkle-native clients (InputFeature/Trajectory I/O).""" self._normalize_ref_outputs(kwargs) diff --git a/src/twinkle/utils/nccl_safe.py b/src/twinkle/utils/nccl_safe.py index 8bafeb1f..d6633355 100644 --- a/src/twinkle/utils/nccl_safe.py +++ b/src/twinkle/utils/nccl_safe.py @@ -1,339 +1,65 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""NCCL-safe utilities for production distributed training. - -Provides three layers of protection to prevent NCCL hangs: - -Layer 1 - safe_loss(): - Wraps loss instances to catch computation errors and return - graph-connected zero loss (ensures FSDP ReduceScatter can proceed). - -Layer 2 - @nccl_safe decorator: - Wraps forward_backward methods to ensure backward() always executes - after forward() has started, even if intermediate code raises. - -Layer 3 - @nccl_safe_megatron decorator: - Wraps Megatron backend methods (forward_only, forward_backward) where - the entire function body involves NCCL communication (sync=True). - Catches pre-communication errors (e.g. data preprocessing failures) - that would otherwise leave other DP ranks waiting at a collective. - -Controlled by environment variable: - TWINKLE_FAIL_FAST=1 (default, development): all protection is transparent, - exceptions propagate normally. - TWINKLE_FAIL_FAST=0 (production): protection activated, exceptions in - NCCL-critical sections are caught and handled gracefully. +"""NCCL critical-section failure logging. + +Single responsibility: inside a Megatron NCCL-critical method, log a +rank-attributed failure and then re-raise it unchanged. + +This module does NOT prevent asymmetric-failure blocking -- nothing at this layer +can. A rank that swallows its exception still does not enter the collective, so the +other ranks stay blocked regardless. The time bound for an asymmetric failure comes +from Ray_Get_Timeout (the effective execution timeout applied per future), not from +this decorator. Diagnosability is the only reason this wrapper exists. + +Coverage removed together with the former Layer 1 (the loss-instance wrapper) and +Layer 2 (the forward/backward decorator) silent degradation: under FSDP, the window +between ``calculate_loss``'s loss call and its surrounding bookkeeping (metric +accumulation, ``status.num_tokens``), between the three calls inside a +``forward_backward`` body, and numerical problems inside a loss (NaN, shape mismatch) +may each constitute a "forward ran, backward did not" asymmetric-failure window. That +window is no longer covered by any silent degradation; its time bound is the two +bounds documented for the task queue (record-terminal = ``queue_timeout + T``; +resource-release = ``Collect_Width * T``). """ import functools -import os -from twinkle.data_format import LossOutput -from twinkle.loss import Loss from twinkle.utils.logger import get_logger logger = get_logger() +# Errors are logged with at most this many trailing characters of traceback. +_TRACEBACK_LIMIT = 8192 -def _is_fail_fast() -> bool: - """Check if fail-fast mode is enabled (default: enabled). - - Returns True (fail-fast/development mode) unless TWINKLE_FAIL_FAST - is explicitly set to a falsy value. - """ - val = os.getenv('TWINKLE_FAIL_FAST', '1').upper() - return val not in ('0', 'NO', 'FALSE', 'OFF') - - -# ─── Layer 1: safe_loss ──────────────────────────────────────────────────── +def _global_rank() -> int: + """Best-effort global rank for failure attribution; -1 if unavailable.""" + try: + from twinkle.utils import Platform + return Platform.get_rank() + except Exception: + return -1 -def safe_loss(loss_instance): - """Wrap loss instance for production graceful degradation. - Always wraps the loss instance (idempotent). The fail-fast check is deferred - to call time so that TWINKLE_FAIL_FAST can be set after wrapping (e.g. in - Ray actor processes where env vars may not be inherited from the launcher). +def nccl_safe_megatron(func): + """Log a rank-attributed failure inside the NCCL critical section, then re-raise. - When TWINKLE_FAIL_FAST=1 (default, development): wrapper is transparent, - exceptions propagate normally. - When TWINKLE_FAIL_FAST=0 (production): wrapper catches exceptions and - returns a graph-connected zero loss (ensures FSDP ReduceScatter proceeds). - - Idempotent: already-wrapped instances are returned as-is. - """ - if getattr(loss_instance, '_nccl_safe_wrapped', False): - return loss_instance - return SafeLossWrapper(loss_instance) - - -class SafeLossWrapper(Loss): - """Loss subclass that catches computation errors and returns graph-connected zero loss. - - Inherits from :class:`twinkle.loss.Loss` so ``isinstance(wrapper, Loss)`` - assertions in the training pipeline continue to pass. + This decorator does *not* prevent asymmetric-failure blocking -- nothing at this + layer can. A rank that swallows its exception still does not enter the collective. + The time bound for that case comes from Ray_Get_Timeout. Diagnosability is the only + reason this wrapper still exists. Its behavior is unconditional: no environment + variable or config switch affects it, and it returns no degraded value. """ - def __init__(self, loss_instance): - super().__init__() - self._loss_instance = loss_instance - self.require_logps = getattr(loss_instance, 'require_logps', True) - self.require_entropy = getattr(loss_instance, 'require_entropy', False) - self.require_logits = getattr(loss_instance, 'require_logits', False) - self.enable_sampling_replay = getattr(loss_instance, 'enable_sampling_replay', False) - self.require_values = getattr(loss_instance, 'require_values', False) - self.reduction = getattr(loss_instance, 'reduction', 'mean') - self._nccl_safe_wrapped = True - - def __call__(self, inputs, outputs, **kwargs): - if _is_fail_fast(): - return self._loss_instance(inputs, outputs, **kwargs) + @functools.wraps(func) + def wrapper(self, *args, **kwargs): try: - return self._loss_instance(inputs, outputs, **kwargs) - except Exception as e: + return func(self, *args, **kwargs) + except Exception as exc: import traceback - logger.warning('[nccl_safe] Loss computation skipped due to error: ' - '%s: %s\n%s', - type(e).__name__, e, traceback.format_exc()) - return _zero_loss(outputs) - - def micro_batch_scale(self, inputs, indices): - """Preserve the wrapped loss's micro-batch reduction semantics.""" - return self._loss_instance.micro_batch_scale(inputs, indices) - - -def _zero_loss(outputs) -> 'LossOutput': - """Create a graph-connected zero loss for FSDP compatibility. - - Finds a gradient-bearing tensor from outputs to maintain graph connectivity, - ensuring backward hooks (ReduceScatter) fire. - """ - import torch - if isinstance(outputs, dict): - for key in ('logps', 'values', 'logits', 'loss'): - t = outputs.get(key) - if t is not None and isinstance(t, torch.Tensor) and t.requires_grad: - return LossOutput(loss=(t.flatten()[:1] * 0).sum(), num_tokens=0) - # Fallback: standalone zero tensor (may not trigger FSDP hooks) - device = 'cpu' - if isinstance(outputs, dict): - for v in outputs.values(): - if hasattr(v, 'device'): - device = v.device - break - return LossOutput(loss=torch.zeros((), device=device, requires_grad=True), num_tokens=0) - - -# ─── Layer 2: @nccl_safe decorator ────────────────────────────────────────── - - -def nccl_safe(func=None, *, tinker=False): - """Decorator ensuring backward() executes if forward() has already run. - - Detects forward completion by comparing train_status.outputs before/after - the wrapped function call. If an exception occurs after forward has run - but before backward completes, forces a zero-gradient backward pass to - prevent NCCL hang (other ranks waiting for ReduceScatter). - - Args: - func: The function to decorate (when used without arguments). - tinker: If True, fallback returns ``[[], 0.0]`` (tinker format). - If False, fallback returns outputs dict with ``loss=0.0``. - - Usage:: - - @remote_function(dispatch='slice_dp', collect=...) - @nccl_safe(tinker=True) - def tinker_forward_backward(self, *, inputs, adapter_name, ...): - # method body completely unchanged - ... - - @remote_function(dispatch='slice_dp', collect=...) - @nccl_safe - def forward_backward(self, *, inputs, **kwargs): - # method body completely unchanged - ... - """ - - def decorator(fn): - - @functools.wraps(fn) - def wrapper(self, *args, **kwargs): - if _is_fail_fast(): - return fn(self, *args, **kwargs) - - # Extract adapter_name for state tracking - adapter_name = kwargs.get('adapter_name') - if adapter_name is None and hasattr(self, '_get_default_group'): - adapter_name = self._get_default_group() - - og = self.optimizer_group.get(adapter_name) if adapter_name else None - if og is None: - # Cannot track state without optimizer group, passthrough - return fn(self, *args, **kwargs) - - # Snapshot state before call to detect forward completion - outputs_before = og.train_status.outputs - - try: - return fn(self, *args, **kwargs) - except Exception as e: - outputs_after = og.train_status.outputs - forward_ran = (outputs_after is not None and outputs_after is not outputs_before) - - if not forward_ran: - # Pre-forward failure: no NCCL ops started, safe to propagate - raise - - # Forward completed. Check if backward already ran. - # TransformersModel.backward() clears loss_value to None. - backward_done = (og.train_status.loss_value is None) - - if backward_done: - # Post-backward failure (e.g. output formatting) - # No NCCL hang risk, just return gracefully - logger.warning(f'[nccl_safe] Post-backward error (no NCCL risk): ' - f'{type(e).__name__}: {e}') - else: - # CRITICAL: forward ran but backward didn't → NCCL hang risk! - logger.warning(f'[nccl_safe] Forcing zero backward to prevent NCCL hang: ' - f'{type(e).__name__}: {e}') - _force_zero_backward(self, og, adapter_name, kwargs) - - # Return fallback result - if tinker: - return [[], 0.0] - outputs_after['loss'] = 0.0 - return outputs_after - - return wrapper - - if func is not None: - # @nccl_safe without arguments - return decorator(func) - # @nccl_safe(tinker=True) with arguments - return decorator - - -def _iter_model_params(model): - """Iterate parameters from ``model.model``, supporting single model or list of models.""" - raw_model = getattr(model, 'model', None) - if raw_model is None: - return iter([]) - if isinstance(raw_model, (list, tuple)): - for m in raw_model: - yield from m.parameters() - else: - yield from raw_model.parameters() - - -def _force_zero_backward(model, og, adapter_name, kwargs): - """Force a zero-gradient backward pass to prevent NCCL hang. - - Creates a graph-connected zero loss tensor and calls backward(), - ensuring FSDP ReduceScatter hooks fire on all ranks. - """ - import torch - - outputs = og.train_status.outputs - - # Find a graph-connected tensor for zero loss - zero_loss = None - if outputs is not None and isinstance(outputs, dict): - for key in ('logps', 'values', 'logits', 'loss'): - t = outputs.get(key) - if t is not None and isinstance(t, torch.Tensor) and t.requires_grad: - zero_loss = (t.flatten()[:1] * 0).sum() - break - - if zero_loss is None: - # Fallback: use first model parameter to maintain graph connectivity. - # Do NOT detach() the parameter -- the zero loss must remain connected - # to the model's autograd graph so FSDP ReduceScatter hooks fire. - # Use lazy iteration to avoid materializing the full parameter list. - try: - param = next((p for p in _iter_model_params(model) if p.requires_grad), None) - if param is not None: - zero_loss = (param.flatten()[0] * 0).sum() - else: - zero_loss = torch.zeros((), device='cuda', requires_grad=True) - except Exception: - zero_loss = torch.zeros((), device='cuda', requires_grad=True) - - og.train_status.loss_value = zero_loss - - # Call backward with minimal kwargs - bwd_kwargs = {'adapter_name': adapter_name} - gas = kwargs.get('gradient_accumulation_steps') - if gas is not None: - bwd_kwargs['gradient_accumulation_steps'] = gas - model.backward(**bwd_kwargs) - - -# ─── Layer 3: @nccl_safe_megatron decorator ────────────────────────────────── - - -def nccl_safe_megatron(func=None, *, tinker=False, forward_only=False): - """Decorator for Megatron backend methods where the entire body is NCCL-critical. - - Unlike @nccl_safe (which detects forward/backward boundaries), this decorator - treats the **entire function** as a NCCL-critical section. In Megatron, - forward_only and forward_backward both call get_forward_backward_func() which - requires all DP ranks to enter synchronously. If one rank fails during data - preprocessing (before entering Megatron's scheduler), other ranks will hang - waiting for the collective. - - This decorator catches ALL exceptions (when TWINKLE_FAIL_FAST=0) and returns - a safe fallback value, preventing NCCL hang from asymmetric failures. - - Args: - func: The function to decorate (when used without arguments). - tinker: If True, fallback returns ``[[], 0.0]`` (tinker format). - forward_only: If True, fallback returns empty dict ``{}`` (forward_only format). - - Usage:: - - @remote_function(dispatch='slice_dp', collect=..., sync=True) - @nccl_safe_megatron - def forward_backward(self, *, inputs, **kwargs): - ... - - @remote_function(dispatch='slice_dp', collect=...) - @nccl_safe_megatron(forward_only=True) - def forward_only(self, *, inputs, **kwargs): - ... - - @remote_function(dispatch='slice_dp', collect=..., sync=True) - @nccl_safe_megatron(tinker=True) - def tinker_forward_backward(self, *, inputs, **kwargs): - ... - """ - - def decorator(fn): - - @functools.wraps(fn) - def wrapper(self, *args, **kwargs): - if _is_fail_fast(): - return fn(self, *args, **kwargs) - - try: - return fn(self, *args, **kwargs) - except Exception as e: - import traceback - logger.warning(f'[nccl_safe_megatron] Exception in Megatron method ' - f'{fn.__name__}: {type(e).__name__}: {e}\n' - f'{traceback.format_exc()}') - - # Return safe fallback to prevent NCCL hang on other ranks - if tinker: - return [[], 0.0] - if forward_only: - return {} - # forward_backward fallback: return dict with loss=0.0 - return {'loss': 0.0} - - return wrapper - - if func is not None: - # @nccl_safe_megatron without arguments - return decorator(func) - # @nccl_safe_megatron(tinker=True) with arguments - return decorator + tb = traceback.format_exc() + if len(tb) > _TRACEBACK_LIMIT: + tb = tb[-_TRACEBACK_LIMIT:] + logger.error('[nccl_safe_megatron] %s in %s on global rank %s:\n%s', type(exc).__name__, func.__name__, + _global_rank(), tb) + raise + + return wrapper diff --git a/tests/model/test_micro_batch.py b/tests/model/test_micro_batch.py index 8d21c1db..ec4083e9 100644 --- a/tests/model/test_micro_batch.py +++ b/tests/model/test_micro_batch.py @@ -8,7 +8,6 @@ from twinkle.model.micro_batch import MicroBatchConfig, plan_micro_batches from twinkle.model.transformers.transformers import TransformersModel from twinkle.processor import InputProcessor -from twinkle.utils.nccl_safe import safe_loss @pytest.mark.parametrize('packing_algorithm', ['ffd', 'kk']) @@ -72,17 +71,6 @@ def test_sample_mean_and_token_sum_micro_batch_scales(): assert CrossEntropyLoss(reduction='sum').micro_batch_scale(inputs, [0]) == 1.0 -def test_safe_loss_preserves_wrapped_micro_batch_scale(): - inputs = [ - {'labels': [1, -100]}, - {'labels': [2, 3]}, - {'labels': [4, -100]}, - {'labels': [5, 6]}, - ] - - assert safe_loss(GRPOLoss()).micro_batch_scale(inputs, [0, 2]) == .5 - - def test_loss_without_micro_batch_semantics_fails_when_split(): with pytest.raises(NotImplementedError, match='does not support micro-batching'): Loss().micro_batch_scale([{}, {}], [0]) diff --git a/tests/server/integration/test_nccl_safe_tinker_e2e.py b/tests/server/integration/test_nccl_safe_tinker_e2e.py index aa94f99c..b47666b9 100644 --- a/tests/server/integration/test_nccl_safe_tinker_e2e.py +++ b/tests/server/integration/test_nccl_safe_tinker_e2e.py @@ -1,16 +1,15 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Real E2E test for NCCL-safe fault tolerance via Tinker client path. +"""Real E2E test for loud-failure semantics via the Tinker client path. -Exercises the /tinker/forward_backward endpoint through the upstream Tinker SDK. -All adversarial scenarios verify that safe_loss catches errors gracefully without -NCCL hang or model state corruption. +Exercises ``/tinker/forward_backward`` through the upstream Tinker SDK. The +invariant under test (post silent-degradation removal): a request whose loss +computation fails does NOT come back as a silent zero-loss success -- it enters a +failed terminal state -- and a subsequent valid request on the same deployment +still succeeds. Prerequisites: - 1. Ray cluster running with GPUs (2 for model DP/TP, optionally 1 for sampler) - 2. Twinkle server started with TWINKLE_FAIL_FAST=0 - -Usage (direct): - python tests/server/integration/test_nccl_safe_tinker_e2e.py + 1. Ray cluster running with GPUs (2 for model DP/TP) + 2. Twinkle server started (no fault-tolerance env switch exists any more) Usage (pytest, requires TWINKLE_TEST_GPU_E2E=1): TWINKLE_TEST_GPU_E2E=1 pytest tests/server/integration/test_nccl_safe_tinker_e2e.py -v @@ -18,10 +17,7 @@ from __future__ import annotations import os -import sys import time -import logging -import traceback import numpy as np import pytest @@ -31,409 +27,61 @@ reason='Set TWINKLE_TEST_GPU_E2E=1 to run real GPU E2E tests (requires running server)', ) -logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') -logger = logging.getLogger(__name__) - - -def log(msg): - """Print + flush to avoid log suppression by init_tinker_client().""" - print(f'[E2E-Tinker] {msg}', flush=True) - - BASE_MODEL = 'Qwen/Qwen3.5-4B' SERVER_URL = os.environ.get('TWINKLE_SERVER_URL', 'http://localhost:9000') TIMEOUT = 120 -def wait_for_server(url, timeout=300): - """Wait for Twinkle server to become ready.""" - import requests - start = time.time() - while time.time() - start < timeout: - try: - resp = requests.get(f'{url}/-/routes', timeout=5) - if resp.status_code == 200: - elapsed = int(time.time() - start) - log(f'Server is ready (waited {elapsed}s)') - return True - except Exception: - pass - time.sleep(5) - raise TimeoutError(f'Server not ready after {timeout}s') - - -def init_client(): - """Initialize Tinker client and create training client.""" +def _init_client(): os.environ['TINKER_BASE_URL'] = SERVER_URL os.environ['TWINKLE_SERVER_TOKEN'] = 'EMPTY_TOKEN' - from twinkle_client import init_tinker_client init_tinker_client() - from tinker import ServiceClient - service_client = ServiceClient() - training_client = service_client.create_lora_training_client(base_model=BASE_MODEL, rank=16) - log('Training client created successfully') - return training_client + return ServiceClient().create_lora_training_client(base_model=BASE_MODEL, rank=16) -def make_datum(seq_len=32, completion_len=16, *, bad_logprobs_len=None, include_advantages=True): - """Construct a Datum for GRPO training.""" +def _make_datum(seq_len=64, completion_len=32, *, bad_logprobs_len=None): from tinker import types - prompt_len = seq_len - completion_len input_tokens = list(range(1, seq_len + 1)) target_tokens = [0] * prompt_len + list(range(100, 100 + completion_len)) weights = [0] * prompt_len + [1] * completion_len - - if bad_logprobs_len is not None: - logprobs_values = np.random.randn(bad_logprobs_len).astype(np.float32) - padded_logprobs = [0.0] * prompt_len + logprobs_values.tolist() - else: - logprobs_values = np.random.randn(completion_len).astype(np.float32) - padded_logprobs = [0.0] * prompt_len + logprobs_values.tolist() - - loss_fn_inputs = { - 'target_tokens': target_tokens, - 'weights': weights, - 'logprobs': types.TensorData.from_numpy(np.array(padded_logprobs, dtype=np.float32)), - } - - if include_advantages: - advantage = float(np.random.randn()) - padded_advantages = [0.0] * prompt_len + [advantage] * completion_len - loss_fn_inputs['advantages'] = types.TensorData.from_numpy( - np.array(padded_advantages, dtype=np.float32)) - + n = bad_logprobs_len if bad_logprobs_len is not None else completion_len + padded_logprobs = [0.0] * prompt_len + np.random.randn(n).astype(np.float32).tolist() + advantage = float(np.random.randn()) return types.Datum( model_input=types.ModelInput.from_ints(input_tokens), - loss_fn_inputs=loss_fn_inputs, + loss_fn_inputs={ + 'target_tokens': target_tokens, + 'weights': weights, + 'logprobs': types.TensorData.from_numpy(np.array(padded_logprobs, dtype=np.float32)), + 'advantages': types.TensorData.from_numpy( + np.array([0.0] * prompt_len + [advantage] * completion_len, dtype=np.float32)), + }, ) -def run_forward_backward(training_client, datums, test_name, expect_success=True): - """Run forward_backward and return (success, result, elapsed_seconds).""" - log(f'[{test_name}] Sending {len(datums)} datums...') - start = time.time() - try: - result = training_client.forward_backward(datums, 'importance_sampling').result() - elapsed = time.time() - start - log(f'[{test_name}] Completed in {elapsed:.1f}s') - if hasattr(result, 'metrics') and result.metrics: - loss_avg = result.metrics.get('loss:avg', 'N/A') - log(f'[{test_name}] loss:avg = {loss_avg}') - return True, result, elapsed - except Exception as e: - elapsed = time.time() - start - log(f'[{test_name}] FAILED in {elapsed:.1f}s: {type(e).__name__}: {e}') - if elapsed > TIMEOUT: - log(f'[{test_name}] TIMEOUT! This suggests NCCL hang!') - return False, None, elapsed - - -def do_optim_step(training_client, test_name): - """Run optimizer step.""" - from tinker import types - try: - training_client.optim_step(types.AdamParams(learning_rate=1e-5)).result() - log(f'[{test_name}] optim_step OK') - return True - except Exception as e: - log(f'[{test_name}] optim_step FAILED: {e}') - return False - - -# ═══════════════════════════════════════════════════════════════════════════ -# Test Scenarios (19 tests) -# ═══════════════════════════════════════════════════════════════════════════ - -def test_1_normal_grpo(tc): - datums = [make_datum(seq_len=64, completion_len=32) for _ in range(4)] - ok, result, elapsed = run_forward_backward(tc, datums, 'TEST-1-NORMAL') - assert ok and elapsed < TIMEOUT - do_optim_step(tc, 'TEST-1-NORMAL') - return True - -def test_2_bad_old_logps(tc): - datums = [ - make_datum(seq_len=64, completion_len=32), - make_datum(seq_len=64, completion_len=32, bad_logprobs_len=5), - make_datum(seq_len=64, completion_len=32), - make_datum(seq_len=64, completion_len=32, bad_logprobs_len=99), - ] - ok, result, elapsed = run_forward_backward(tc, datums, 'TEST-2-BAD-LOGPS') - if not ok: - return elapsed < TIMEOUT - assert elapsed < TIMEOUT - do_optim_step(tc, 'TEST-2-BAD-LOGPS') - return True - -def test_3_recovery(tc): - datums = [make_datum(seq_len=64, completion_len=32) for _ in range(4)] - ok, _, elapsed = run_forward_backward(tc, datums, 'TEST-3-RECOVERY') - assert ok and elapsed < TIMEOUT - do_optim_step(tc, 'TEST-3-RECOVERY') - return True - -def test_4_no_advantages(tc): - datums = [make_datum(seq_len=64, completion_len=32, include_advantages=False) for _ in range(4)] - ok, _, elapsed = run_forward_backward(tc, datums, 'TEST-4-NO-ADV') - assert ok and elapsed < TIMEOUT - do_optim_step(tc, 'TEST-4-NO-ADV') - return True - -def test_5_consecutive_bad(tc): - for i in range(5): - datums = [make_datum(seq_len=64, completion_len=32, bad_logprobs_len=3+i) for _ in range(4)] - _, _, elapsed = run_forward_backward(tc, datums, f'TEST-5-{i+1}') - if elapsed >= TIMEOUT: - return False - do_optim_step(tc, f'TEST-5-{i+1}') - return True +def test_failure_is_terminal_then_valid_request_succeeds(): + """A malformed request fails loudly (terminal), a subsequent valid one succeeds. -def test_6_nan_logprobs(tc): + Replaces the former assertion "failure degraded to zero loss and training + continued". If the recovery request does not reach a terminal success, that is + recorded as evidence that R3#2-3 actor recovery and R2#3-4 admission gate are + necessary, not optional. + """ from tinker import types - datums = [] - for _ in range(4): - d = make_datum(seq_len=64, completion_len=32) - d.loss_fn_inputs['logprobs'] = types.TensorData.from_numpy( - np.array([float('nan')] * 64, dtype=np.float32)) - datums.append(d) - _, _, elapsed = run_forward_backward(tc, datums, 'TEST-6-NAN') - if elapsed >= TIMEOUT: - return False - do_optim_step(tc, 'TEST-6-NAN') - return True + tc = _init_client() -def test_7_inf_logprobs(tc): - from tinker import types - datums = [] - for _ in range(4): - d = make_datum(seq_len=64, completion_len=32) - inf_arr = np.full(64, float('inf'), dtype=np.float32) - inf_arr[::2] = float('-inf') - d.loss_fn_inputs['logprobs'] = types.TensorData.from_numpy(inf_arr) - datums.append(d) - _, _, elapsed = run_forward_backward(tc, datums, 'TEST-7-INF') - if elapsed >= TIMEOUT: - return False - do_optim_step(tc, 'TEST-7-INF') - return True - -def test_8_extreme_advantages(tc): - from tinker import types - datums = [] - for i in range(4): - d = make_datum(seq_len=64, completion_len=32) - val = 1e30 if i % 2 == 0 else -1e30 - adv = np.full(64, 0.0, dtype=np.float32) - adv[32:] = val - d.loss_fn_inputs['advantages'] = types.TensorData.from_numpy(adv) - datums.append(d) - _, _, elapsed = run_forward_backward(tc, datums, 'TEST-8-EXTREME-ADV') - if elapsed >= TIMEOUT: - return False - do_optim_step(tc, 'TEST-8-EXTREME-ADV') - return True - -def test_9_zero_completion(tc): - from tinker import types - datums = [] - for _ in range(4): - d = types.Datum( - model_input=types.ModelInput.from_ints(list(range(1, 65))), - loss_fn_inputs={ - 'target_tokens': [0]*64, 'weights': [0]*64, - 'logprobs': types.TensorData.from_numpy(np.zeros(64, dtype=np.float32)), - 'advantages': types.TensorData.from_numpy(np.zeros(64, dtype=np.float32)), - }, - ) - datums.append(d) - _, _, elapsed = run_forward_backward(tc, datums, 'TEST-9-ZERO-COMPL') - if elapsed >= TIMEOUT: - return False - do_optim_step(tc, 'TEST-9-ZERO-COMPL') - return True - -def test_10_partial_advantages(tc): - datums = [ - make_datum(seq_len=64, completion_len=32, include_advantages=True), - make_datum(seq_len=64, completion_len=32, include_advantages=False), - make_datum(seq_len=64, completion_len=32, include_advantages=True), - make_datum(seq_len=64, completion_len=32, include_advantages=False), - ] - _, _, elapsed = run_forward_backward(tc, datums, 'TEST-10-PARTIAL-ADV') - if elapsed >= TIMEOUT: - return False - do_optim_step(tc, 'TEST-10-PARTIAL-ADV') - return True - -def test_11_mixed_seq_lengths(tc): - datums = [ - make_datum(seq_len=32, completion_len=16), - make_datum(seq_len=128, completion_len=64), - make_datum(seq_len=48, completion_len=24), - make_datum(seq_len=96, completion_len=48), - ] - _, _, elapsed = run_forward_backward(tc, datums, 'TEST-11-MIXED') - if elapsed >= TIMEOUT: - return False - do_optim_step(tc, 'TEST-11-MIXED') - return True - -def test_12_all_bad(tc): - datums = [make_datum(seq_len=64, completion_len=32, bad_logprobs_len=i) for i in range(4)] - _, _, elapsed = run_forward_backward(tc, datums, 'TEST-12-ALL-BAD') - if elapsed >= TIMEOUT: - return False - do_optim_step(tc, 'TEST-12-ALL-BAD') - return True - -def test_13_forward_only_then_train(tc): - datums_infer = [make_datum(seq_len=64, completion_len=32, include_advantages=False) for _ in range(4)] + # Deliberately malformed: logprobs length inconsistent with the completion. + bad = [_make_datum(bad_logprobs_len=5) for _ in range(4)] start = time.time() - try: - tc.forward(datums_infer).result() - except Exception: - if time.time() - start >= TIMEOUT: - return False - datums_train = [make_datum(seq_len=64, completion_len=32) for _ in range(4)] - ok, _, elapsed = run_forward_backward(tc, datums_train, 'TEST-13-TRAIN') - if not ok or elapsed >= TIMEOUT: - return False - do_optim_step(tc, 'TEST-13-TRAIN') - return True - -def test_14_rapid_bad_good(tc): - for i in range(5): - bad = [make_datum(seq_len=64, completion_len=32, bad_logprobs_len=i+1) for _ in range(4)] - _, _, elapsed = run_forward_backward(tc, bad, f'TEST-14-BAD-{i+1}') - if elapsed >= TIMEOUT: - return False - do_optim_step(tc, f'TEST-14-BAD-{i+1}') - good = [make_datum(seq_len=64, completion_len=32) for _ in range(4)] - ok, _, elapsed = run_forward_backward(tc, good, f'TEST-14-GOOD-{i+1}') - if not ok or elapsed >= TIMEOUT: - return False - do_optim_step(tc, f'TEST-14-GOOD-{i+1}') - return True - -def test_15_final_health(tc): - datums = [make_datum(seq_len=64, completion_len=32) for _ in range(4)] - ok, _, elapsed = run_forward_backward(tc, datums, 'TEST-15-FINAL') - assert ok and elapsed < TIMEOUT - do_optim_step(tc, 'TEST-15-FINAL') - return True - -def test_16_large_batch(tc): - datums = [make_datum(seq_len=64, completion_len=32) for _ in range(16)] - ok, _, elapsed = run_forward_backward(tc, datums, 'TEST-16-LARGE') - if elapsed >= TIMEOUT: - return False - assert ok - do_optim_step(tc, 'TEST-16-LARGE') - return True - -def test_17_single_datum(tc): - # With dp_size=2 + nproc_per_node=2, minimum batch must be >= data_world_size - datums = [make_datum(seq_len=64, completion_len=32) for _ in range(4)] - ok, _, elapsed = run_forward_backward(tc, datums, 'TEST-17-SMALL') - if elapsed >= TIMEOUT: - return False - assert ok - do_optim_step(tc, 'TEST-17-SMALL') - return True - -def test_18_save_after_error(tc): - bad = [make_datum(seq_len=64, completion_len=32, bad_logprobs_len=2) for _ in range(4)] - _, _, elapsed = run_forward_backward(tc, bad, 'TEST-18-ERR') - if elapsed >= TIMEOUT: - return False - do_optim_step(tc, 'TEST-18-ERR') - try: - tc.save_weights_for_sampler().result() - except Exception: - pass - good = [make_datum(seq_len=64, completion_len=32) for _ in range(4)] - ok, _, elapsed = run_forward_backward(tc, good, 'TEST-18-POST') - if not ok or elapsed >= TIMEOUT: - return False - do_optim_step(tc, 'TEST-18-POST') - return True - -def test_19_consecutive_optim_steps(tc): - datums = [make_datum(seq_len=64, completion_len=32) for _ in range(4)] - ok, _, elapsed = run_forward_backward(tc, datums, 'TEST-19-BASE') - assert ok and elapsed < TIMEOUT - for i in range(3): - do_optim_step(tc, f'TEST-19-STEP-{i+1}') - datums = [make_datum(seq_len=64, completion_len=32) for _ in range(4)] - ok, _, elapsed = run_forward_backward(tc, datums, 'TEST-19-VERIFY') - if not ok or elapsed >= TIMEOUT: - return False - do_optim_step(tc, 'TEST-19-VERIFY') - return True - - -ALL_TESTS = [ - ('TEST-1: Normal GRPO Training', test_1_normal_grpo), - ('TEST-2: Bad old_logps (original bug)', test_2_bad_old_logps), - ('TEST-3: Recovery after error', test_3_recovery), - ('TEST-4: No advantages (zero loss)', test_4_no_advantages), - ('TEST-5: Consecutive bad batches', test_5_consecutive_bad), - ('TEST-6: NaN logprobs', test_6_nan_logprobs), - ('TEST-7: +Inf/-Inf logprobs', test_7_inf_logprobs), - ('TEST-8: Extreme advantages (1e30)', test_8_extreme_advantages), - ('TEST-9: Zero completion tokens', test_9_zero_completion), - ('TEST-10: Partial advantages (ragged)', test_10_partial_advantages), - ('TEST-11: Mixed sequence lengths', test_11_mixed_seq_lengths), - ('TEST-12: All datums bad (100%)', test_12_all_bad), - ('TEST-13: forward_only then train', test_13_forward_only_then_train), - ('TEST-14: Rapid bad->good alternation', test_14_rapid_bad_good), - ('TEST-15: Final health check', test_15_final_health), - ('TEST-16: Large batch (16 datums)', test_16_large_batch), - ('TEST-17: Single datum batch', test_17_single_datum), - ('TEST-18: Save after error', test_18_save_after_error), - ('TEST-19: Consecutive optim_steps', test_19_consecutive_optim_steps), -] - - -def main(): - log('=' * 60) - log('NCCL-Safe E2E Test - Tinker Client Path') - log('=' * 60) - log(f'Server URL: {SERVER_URL}') - log(f'Base Model: {BASE_MODEL}') - log(f'TWINKLE_FAIL_FAST = {os.getenv("TWINKLE_FAIL_FAST", "1 (default)")}') - - wait_for_server(SERVER_URL) - tc = init_client() - - results = [] - for name, test_fn in ALL_TESTS: - log(f'\n{"=" * 60}\n{name}\n{"=" * 60}') - try: - passed = test_fn(tc) - results.append((name, 'PASS' if passed else 'FAIL')) - log(f'[{name}] {"PASS" if passed else "FAIL"}') - except Exception as e: - log(f'{name}: EXCEPTION: {e}') - traceback.print_exc() - results.append((name, 'FAIL')) - - log(f'\n{"=" * 60}\nRESULTS SUMMARY\n{"=" * 60}') - all_passed = all(s == 'PASS' for _, s in results) - for name, status in results: - log(f' [{status}] {name}') - log(f'\n{"ALL" if all_passed else "SOME"} {len(results)} TESTS {"PASSED" if all_passed else "FAILED"}!') - return 0 if all_passed else 1 - - -def test_nccl_safe_tinker_e2e(): - """Pytest-collected entry point.""" - rc = main() - assert rc == 0, 'Some Tinker NCCL-safe E2E tests failed' - - -if __name__ == '__main__': - sys.exit(main()) + with pytest.raises(Exception): # RequestFailedError or a raised failed terminal + tc.forward_backward(bad, 'importance_sampling').result() + assert time.time() - start < TIMEOUT, 'malformed request must fail fast, not hang (NCCL)' + + # Recovery: a subsequent valid request on the same deployment must succeed. + good = [_make_datum() for _ in range(4)] + result = tc.forward_backward(good, 'importance_sampling').result() + assert result is not None + tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() diff --git a/tests/server/integration/test_nccl_safe_twinkle_e2e.py b/tests/server/integration/test_nccl_safe_twinkle_e2e.py index c9cce48a..90a33cf3 100644 --- a/tests/server/integration/test_nccl_safe_twinkle_e2e.py +++ b/tests/server/integration/test_nccl_safe_twinkle_e2e.py @@ -1,16 +1,14 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Real E2E test for NCCL-safe fault tolerance via Twinkle client path. +"""Real E2E test for loud-failure semantics via the Twinkle-native client path. -Exercises the /twinkle/forward_backward endpoint through the Twinkle SDK -(init_twinkle_client + MultiLoraTransformersModel). This is a SEPARATE code -path from the Tinker SDK (/tinker/forward_backward). +Exercises ``/twinkle/forward_backward`` through the Twinkle client. The invariant +under test (post silent-degradation removal): a request whose loss computation fails +does NOT come back as a silent zero-loss success -- it fails loudly -- and a +subsequent valid request on the same deployment still succeeds. Prerequisites: - 1. Ray cluster running with GPUs (2 for model DP/TP, optionally 1 for sampler) - 2. Twinkle server started with TWINKLE_FAIL_FAST=0 - -Usage (direct): - python tests/server/integration/test_nccl_safe_twinkle_e2e.py + 1. Ray cluster running with GPUs (2 for model DP/TP) + 2. Twinkle server started (no fault-tolerance env switch exists any more) Usage (pytest, requires TWINKLE_TEST_GPU_E2E=1): TWINKLE_TEST_GPU_E2E=1 pytest tests/server/integration/test_nccl_safe_twinkle_e2e.py -v @@ -18,11 +16,7 @@ from __future__ import annotations import os -import sys import time -import logging -import traceback -from typing import Any, Dict, List import numpy as np import pytest @@ -32,44 +26,18 @@ reason='Set TWINKLE_TEST_GPU_E2E=1 to run real GPU E2E tests (requires running server)', ) -logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s') -logger = logging.getLogger(__name__) - - -def log(msg): - print(f'[E2E-Twinkle] {msg}', flush=True) - - BASE_MODEL = 'Qwen/Qwen3.5-4B' SERVER_URL = os.environ.get('TWINKLE_SERVER_URL', 'http://localhost:9000') TIMEOUT = 120 -ADAPTER_NAME = 'nccl-safe-test' +ADAPTER_NAME = 'loud-failure-test' -def wait_for_server(url, timeout=300): - """Wait for Twinkle server to become ready.""" - import requests - start = time.time() - while time.time() - start < timeout: - try: - resp = requests.get(f'{url}/-/routes', timeout=5) - if resp.status_code == 200: - log(f'Server is ready (waited {int(time.time() - start)}s)') - return True - except Exception: - pass - time.sleep(5) - raise TimeoutError(f'Server not ready after {timeout}s') - - -def init_client(): - """Initialize Twinkle client and configure model for GRPO training.""" +def _init_client(): + from peft import LoraConfig from twinkle_client import init_twinkle_client from twinkle_client.model import MultiLoraTransformersModel - from peft import LoraConfig init_twinkle_client(base_url=SERVER_URL, api_key='EMPTY_TOKEN') - model = MultiLoraTransformersModel(model_id=f'ms://{BASE_MODEL}') model.add_adapter_to_model( adapter_name=ADAPTER_NAME, @@ -80,266 +48,43 @@ def init_client(): model.set_optimizer('Adam', lr=1e-5) model.set_template('Qwen3_5Template') model.set_processor('InputProcessor', padding_side='right') - log('Twinkle client + model configured successfully') return model -def make_input_features( - batch_size=4, seq_len=64, completion_len=32, *, - bad_old_logps_len=None, include_advantages=True, - nan_old_logps=False, extreme_advantages=None, all_labels_masked=False, -): - """Construct InputFeature list + old_logps + advantages for GRPO.""" +def _make_inputs(batch_size=4, seq_len=64, completion_len=32, *, bad_old_logps_len=None): prompt_len = seq_len - completion_len - input_features = [] - old_logps_list = [] - advantages_list = [] - - for i in range(batch_size): - input_ids = list(range(1, seq_len + 1)) - labels = [-100] * seq_len if all_labels_masked else ( - [-100] * prompt_len + list(range(100, 100 + completion_len))) - input_features.append({ - 'input_ids': input_ids, - 'labels': labels, + features, old_logps, advantages = [], [], [] + for _ in range(batch_size): + features.append({ + 'input_ids': list(range(1, seq_len + 1)), + 'labels': [-100] * prompt_len + list(range(100, 100 + completion_len)), 'attention_mask': [1] * seq_len, 'position_ids': list(range(seq_len)), }) - - if bad_old_logps_len is not None: - logps = np.random.randn(bad_old_logps_len).tolist() - elif nan_old_logps: - logps = [float('nan')] * completion_len - else: - logps = np.random.randn(completion_len).tolist() - old_logps_list.append(logps) - - if extreme_advantages is not None: - advantages_list.append(extreme_advantages if i % 2 == 0 else -extreme_advantages) - else: - advantages_list.append(float(np.random.randn())) - - old_logps = old_logps_list if include_advantages else None - advantages = advantages_list if include_advantages else None - return input_features, old_logps, advantages - - -def run_forward_backward(model, inputs, old_logps, advantages, test_name): - """Run forward_backward and return (success, result, elapsed_seconds).""" - log(f'[{test_name}] Sending {len(inputs)} input features...') - start = time.time() - try: - kwargs: Dict[str, Any] = {} - if old_logps is not None: - kwargs['old_logps'] = old_logps - if advantages is not None: - kwargs['advantages'] = advantages - - result = model.forward_backward(inputs=inputs, **kwargs) - elapsed = time.time() - start - log(f'[{test_name}] Completed in {elapsed:.1f}s') - if hasattr(result, 'result') and result.result is not None: - log(f'[{test_name}] result = {result.result}') - return True, result, elapsed - except Exception as e: - elapsed = time.time() - start - log(f'[{test_name}] FAILED in {elapsed:.1f}s: {type(e).__name__}: {e}') - if elapsed > TIMEOUT: - log(f'[{test_name}] TIMEOUT! This suggests NCCL hang!') - return False, None, elapsed - - -def do_optim_step(model, test_name): - """Run clip_grad_and_step.""" - try: - model.clip_grad_and_step() - log(f'[{test_name}] clip_grad_and_step OK') - return True - except Exception as e: - log(f'[{test_name}] clip_grad_and_step FAILED: {e}') - return False + n = bad_old_logps_len if bad_old_logps_len is not None else completion_len + old_logps.append(np.random.randn(n).tolist()) + advantages.append(float(np.random.randn())) + return features, old_logps, advantages -# ═══════════════════════════════════════════════════════════════════════════ -# Test Scenarios (12 tests) -# ═══════════════════════════════════════════════════════════════════════════ +def test_failure_is_terminal_then_valid_request_succeeds(): + """A malformed request fails loudly, a subsequent valid one succeeds. -def test_1_normal_grpo(m): - inputs, old_logps, adv = make_input_features(batch_size=4) - ok, _, elapsed = run_forward_backward(m, inputs, old_logps, adv, 'TEST-1-NORMAL') - assert ok and elapsed < TIMEOUT - do_optim_step(m, 'TEST-1-NORMAL') - return True + Replaces the former assertion "failure degraded to zero loss and training + continued". If the recovery request does not reach a terminal success, that is + recorded as evidence that R3#2-3 actor recovery and R2#3-4 admission gate are + necessary, not optional. + """ + model = _init_client() -def test_2_bad_old_logps(m): - inputs, old_logps, adv = make_input_features(batch_size=4, bad_old_logps_len=5) - ok, _, elapsed = run_forward_backward(m, inputs, old_logps, adv, 'TEST-2-BAD-LOGPS') - if not ok: - return elapsed < TIMEOUT - assert elapsed < TIMEOUT - do_optim_step(m, 'TEST-2-BAD-LOGPS') - return True - -def test_3_recovery(m): - inputs, old_logps, adv = make_input_features(batch_size=4) - ok, _, elapsed = run_forward_backward(m, inputs, old_logps, adv, 'TEST-3-RECOVERY') - assert ok and elapsed < TIMEOUT - do_optim_step(m, 'TEST-3-RECOVERY') - return True - -def test_4_nan_old_logps(m): - inputs, old_logps, adv = make_input_features(batch_size=4, nan_old_logps=True) - _, _, elapsed = run_forward_backward(m, inputs, old_logps, adv, 'TEST-4-NAN') - if elapsed >= TIMEOUT: - return False - do_optim_step(m, 'TEST-4-NAN') - return True - -def test_5_extreme_advantages(m): - inputs, old_logps, adv = make_input_features(batch_size=4, extreme_advantages=1e30) - _, _, elapsed = run_forward_backward(m, inputs, old_logps, adv, 'TEST-5-EXTREME') - if elapsed >= TIMEOUT: - return False - do_optim_step(m, 'TEST-5-EXTREME') - return True - -def test_6_all_labels_masked(m): - inputs, old_logps, adv = make_input_features(batch_size=4, all_labels_masked=True) - _, _, elapsed = run_forward_backward(m, inputs, old_logps, adv, 'TEST-6-MASKED') - if elapsed >= TIMEOUT: - return False - do_optim_step(m, 'TEST-6-MASKED') - return True - -def test_7_consecutive_bad(m): - for i in range(5): - inputs, old_logps, adv = make_input_features(batch_size=4, bad_old_logps_len=i+1) - _, _, elapsed = run_forward_backward(m, inputs, old_logps, adv, f'TEST-7-{i+1}') - if elapsed >= TIMEOUT: - return False - do_optim_step(m, f'TEST-7-{i+1}') - return True - -def test_8_rapid_bad_good(m): - for i in range(5): - bad_in, bad_lp, bad_adv = make_input_features(batch_size=4, bad_old_logps_len=i+1) - _, _, elapsed = run_forward_backward(m, bad_in, bad_lp, bad_adv, f'TEST-8-BAD-{i+1}') - if elapsed >= TIMEOUT: - return False - do_optim_step(m, f'TEST-8-BAD-{i+1}') - good_in, good_lp, good_adv = make_input_features(batch_size=4) - ok, _, elapsed = run_forward_backward(m, good_in, good_lp, good_adv, f'TEST-8-GOOD-{i+1}') - if not ok or elapsed >= TIMEOUT: - return False - do_optim_step(m, f'TEST-8-GOOD-{i+1}') - return True - -def test_9_final_health(m): - inputs, old_logps, adv = make_input_features(batch_size=4) - ok, _, elapsed = run_forward_backward(m, inputs, old_logps, adv, 'TEST-9-FINAL') - assert ok and elapsed < TIMEOUT - do_optim_step(m, 'TEST-9-FINAL') - return True - -def test_10_gradient_accumulation_error(m): - inputs, lp, adv = make_input_features(batch_size=4) - ok, _, elapsed = run_forward_backward(m, inputs, lp, adv, 'TEST-10-GA1') - if not ok or elapsed >= TIMEOUT: - return False - bad_in, bad_lp, bad_adv = make_input_features(batch_size=4, bad_old_logps_len=3) - _, _, elapsed = run_forward_backward(m, bad_in, bad_lp, bad_adv, 'TEST-10-GA2-BAD') - if elapsed >= TIMEOUT: - return False - inputs, lp, adv = make_input_features(batch_size=4) - ok, _, elapsed = run_forward_backward(m, inputs, lp, adv, 'TEST-10-GA3') - if not ok or elapsed >= TIMEOUT: - return False - do_optim_step(m, 'TEST-10-GA') - return True - -def test_11_forward_only_then_train(m): - inputs, _, _ = make_input_features(batch_size=4, include_advantages=False) + bad_features, bad_old_logps, bad_adv = _make_inputs(bad_old_logps_len=5) start = time.time() - try: - m.forward_only(inputs=inputs) - except Exception: - if time.time() - start >= TIMEOUT: - return False - train_in, lp, adv = make_input_features(batch_size=4) - ok, _, elapsed = run_forward_backward(m, train_in, lp, adv, 'TEST-11-TRAIN') - if not ok or elapsed >= TIMEOUT: - return False - do_optim_step(m, 'TEST-11-TRAIN') - return True - -def test_12_mixed_seq_lengths(m): - all_inputs, all_lp, all_adv = [], [], [] - for sl, cl in [(32, 16), (128, 64), (48, 24), (96, 48)]: - feats, lp, adv = make_input_features(batch_size=1, seq_len=sl, completion_len=cl) - all_inputs.extend(feats) - if lp: - all_lp.extend(lp) - if adv: - all_adv.extend(adv) - _, _, elapsed = run_forward_backward(m, all_inputs, all_lp, all_adv, 'TEST-12-MIXED') - if elapsed >= TIMEOUT: - return False - do_optim_step(m, 'TEST-12-MIXED') - return True - - -ALL_TESTS = [ - ('TEST-1: Normal GRPO Training', test_1_normal_grpo), - ('TEST-2: Bad old_logps (original bug)', test_2_bad_old_logps), - ('TEST-3: Recovery after error', test_3_recovery), - ('TEST-4: NaN old_logps', test_4_nan_old_logps), - ('TEST-5: Extreme advantages (1e30)', test_5_extreme_advantages), - ('TEST-6: All labels masked (-100)', test_6_all_labels_masked), - ('TEST-7: Consecutive bad batches', test_7_consecutive_bad), - ('TEST-8: Rapid bad->good', test_8_rapid_bad_good), - ('TEST-9: Final health check', test_9_final_health), - ('TEST-10: Gradient accumulation error', test_10_gradient_accumulation_error), - ('TEST-11: forward_only then train', test_11_forward_only_then_train), - ('TEST-12: Mixed sequence lengths', test_12_mixed_seq_lengths), -] - - -def main(): - log('=' * 60) - log('NCCL-Safe E2E Test - Twinkle Client Path') - log('=' * 60) - log(f'Server URL: {SERVER_URL}') - log(f'Base Model: {BASE_MODEL}') - log(f'TWINKLE_FAIL_FAST = {os.getenv("TWINKLE_FAIL_FAST", "1 (default)")}') - - wait_for_server(SERVER_URL) - m = init_client() - - results = [] - for name, test_fn in ALL_TESTS: - log(f'\n{"=" * 60}\n{name}\n{"=" * 60}') - try: - passed = test_fn(m) - results.append((name, 'PASS' if passed else 'FAIL')) - log(f'[{name}] {"PASS" if passed else "FAIL"}') - except Exception as e: - log(f'{name}: EXCEPTION: {e}') - traceback.print_exc() - results.append((name, 'FAIL')) - - log(f'\n{"=" * 60}\nRESULTS SUMMARY\n{"=" * 60}') - all_passed = all(s == 'PASS' for _, s in results) - for name, status in results: - log(f' [{status}] {name}') - log(f'\n{"ALL" if all_passed else "SOME"} {len(results)} TESTS {"PASSED" if all_passed else "FAILED"}!') - return 0 if all_passed else 1 - - -def test_nccl_safe_twinkle_e2e(): - """Pytest-collected entry point.""" - rc = main() - assert rc == 0, 'Some Twinkle NCCL-safe E2E tests failed' - - -if __name__ == '__main__': - sys.exit(main()) + with pytest.raises(Exception): + model.forward_backward( + inputs=bad_features, adapter_name=ADAPTER_NAME, old_logps=bad_old_logps, advantages=bad_adv) + assert time.time() - start < TIMEOUT, 'malformed request must fail fast, not hang (NCCL)' + + good_features, good_old_logps, good_adv = _make_inputs() + result = model.forward_backward( + inputs=good_features, adapter_name=ADAPTER_NAME, old_logps=good_old_logps, advantages=good_adv) + assert result is not None diff --git a/tests/server/static/test_no_degraded_path.py b/tests/server/static/test_no_degraded_path.py new file mode 100644 index 00000000..765f760a --- /dev/null +++ b/tests/server/static/test_no_degraded_path.py @@ -0,0 +1,68 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Static check: no silent-degradation symbols remain (T7.8 / R6#6 / R9#4 / Property 9). + +One wildcard search covering eight symbols; each must occur zero times in its scope. +The symbols are matched as identifiers (word boundaries) so that ``nccl_safe_megatron`` +(the retained decorator), the ``twinkle.utils.nccl_safe`` module path, and unrelated +test names like ``test_zero_loss_...`` are not counted. +""" +from __future__ import annotations + +import pathlib +import re + +import twinkle + +_TWINKLE_SRC = pathlib.Path(twinkle.__file__).resolve().parent +_REPO_ROOT = _TWINKLE_SRC.parent.parent # .../src/twinkle -> repo root +_TESTS = _REPO_ROOT / 'tests' +_COOKBOOK = _REPO_ROOT / 'cookbook' +_SELF = pathlib.Path(__file__).resolve() + +# symbol -> compiled identifier pattern. +_IDENT = { + 'safe_loss': re.compile(r'(? Date: Tue, 15 Sep 2026 13:55:27 +0800 Subject: [PATCH 09/32] test(contract): zero-wire-change guard + CI/pre-commit closeout (Phase 8, T8.1/T8.2/T8.4) - test_client_api_contract.py: OpenAPI surface == pre-impl baseline (T0.1), schedule_task_and_wait retained, only new client modules base.py/errors.py (T8.1) - new checks are auto-run by CI 'pytest tests'; lint.yaml runs pre-commit --all-files; applied yapf/isort/pyupgrade auto-fixes so all hooks pass on changed src files (T8.2) - tests/server/README.md documents the mock-backend evidence boundary (T8.4) --- src/twinkle/infra/__init__.py | 1 + src/twinkle/infra/_ray/ray_helper.py | 4 +- src/twinkle/model/megatron/megatron.py | 3 +- src/twinkle/server/model/app.py | 4 +- src/twinkle/server/model/tinker_handlers.py | 16 ++++--- src/twinkle/server/model/twinkle_handlers.py | 39 ++++++++++++----- .../server/sampler/twinkle_handlers.py | 12 ++++-- src/twinkle/server/utils/task_errors.py | 6 +-- src/twinkle/server/utils/task_queue/mixin.py | 8 ++-- src/twinkle/server/utils/task_queue/worker.py | 2 +- src/twinkle/utils/nccl_safe.py | 4 +- tests/server/README.md | 23 ++++++++++ .../contract/test_client_api_contract.py | 42 +++++++++++++++++++ 13 files changed, 128 insertions(+), 36 deletions(-) create mode 100644 tests/server/README.md create mode 100644 tests/server/contract/test_client_api_contract.py diff --git a/src/twinkle/infra/__init__.py b/src/twinkle/infra/__init__.py index 3a80219b..2cf7c148 100644 --- a/src/twinkle/infra/__init__.py +++ b/src/twinkle/infra/__init__.py @@ -1047,6 +1047,7 @@ def wrapper(self, *args, **kwargs) -> T1: else: # This is the driver from ._ray import RayHelper + # Resolve the effective ray.get timeout before choosing execute_method: # the decorator's explicit value wins, the instance attribute is the # fallback. ``is not None`` (not ``or``) so that a decorator ``timeout=0`` diff --git a/src/twinkle/infra/_ray/ray_helper.py b/src/twinkle/infra/_ray/ray_helper.py index d3cbc3f5..4cc5f6a6 100644 --- a/src/twinkle/infra/_ray/ray_helper.py +++ b/src/twinkle/infra/_ray/ray_helper.py @@ -137,9 +137,7 @@ def is_worker(): return RayHelper.ray_inited() and ray._private.worker.global_worker.mode == ray._private.worker.WORKER_MODE @staticmethod - def execute_all_sync(method_name: str, - workers_and_args: List[Tuple[Any, List[Any], Dict[str, Any]]], - timeout=None): + def execute_all_sync(method_name: str, workers_and_args: List[Tuple[Any, List[Any], Dict[str, Any]]], timeout=None): """Execute method and return results. ``timeout`` is passed to ``ray.get(list, timeout=)``, whose semantics are diff --git a/src/twinkle/model/megatron/megatron.py b/src/twinkle/model/megatron/megatron.py index 89a05db7..4e4e2bbd 100644 --- a/src/twinkle/model/megatron/megatron.py +++ b/src/twinkle/model/megatron/megatron.py @@ -444,8 +444,7 @@ def forward_step_func(data_iterator, model): batch['labels'] = processor.postprocess_tensor_cp(labels, cu_seqlens=cu_seqlens_q) if completion_mask is not None: # Same index space as labels, so it needs the same CP reassembly. - batch['completion_mask'] = processor.postprocess_tensor_cp( - completion_mask, cu_seqlens=cu_seqlens_q) + batch['completion_mask'] = processor.postprocess_tensor_cp(completion_mask, cu_seqlens=cu_seqlens_q) if 'position_ids' in batch: pos = batch['position_ids'] if pos.dim() == 3: diff --git a/src/twinkle/server/model/app.py b/src/twinkle/server/model/app.py index 6495dd7d..7445ce0d 100644 --- a/src/twinkle/server/model/app.py +++ b/src/twinkle/server/model/app.py @@ -139,7 +139,9 @@ def __init__(self, # ModelManagement opts into the Admission_Gate (collective mis-pairing + # queue-bypassing _cleanup_adapter); SamplerManagement does not. self._init_task_queue( - queue_config, deployment_name='Model', enable_admission_gate=True, + queue_config, + deployment_name='Model', + enable_admission_gate=True, on_backend_timeout=self._probe_after_timeout) # Bound every ray.get on this backend by the effective execution timeout # (applies to both sync=True and sync=False dispatch). T4.1. diff --git a/src/twinkle/server/model/tinker_handlers.py b/src/twinkle/server/model/tinker_handlers.py index 5d95719b..7f34c2ae 100644 --- a/src/twinkle/server/model/tinker_handlers.py +++ b/src/twinkle/server/model/tinker_handlers.py @@ -151,8 +151,7 @@ async def _do_forward(): datum_list = body.forward_input.data loss_fn_config = body.forward_input.loss_fn_config or {} output, loss = await self.call_backend( - self.model.tinker_forward_only, - inputs=datum_list, adapter_name=model_adapter, **loss_fn_config) + self.model.tinker_forward_only, inputs=datum_list, adapter_name=model_adapter, **loss_fn_config) return types.ForwardBackwardOutput( loss_fn_output_type='CrossEntropyLossReturn', loss_fn_outputs=output, @@ -196,7 +195,10 @@ async def _do_forward_backward(): loss_fn_config = body.forward_backward_input.loss_fn_config or {} output, loss = await self.call_backend( self.model.tinker_forward_backward, - inputs=datum_list, adapter_name=model_adapter, loss_fn=loss_fn, **loss_fn_config) + inputs=datum_list, + adapter_name=model_adapter, + loss_fn=loss_fn, + **loss_fn_config) output_type = ('ImportanceSamplingLossReturn' if loss_fn == 'importance_sampling' else 'CrossEntropyLossReturn') self.set_resource_state(adapter_name, 'grad_ready', True) @@ -243,7 +245,8 @@ async def _do_optim(): if not self.get_resource_state(adapter_name, 'grad_ready', False): raise RuntimeError(f'No accumulated gradients for adapter={adapter_name}; ' 'call forward_backward before optim_step') - await self.call_backend(self.model.tinker_step, adam_params=body.adam_params, adapter_name=model_adapter) + await self.call_backend( + self.model.tinker_step, adam_params=body.adam_params, adapter_name=model_adapter) self.set_resource_state(adapter_name, 'grad_ready', False) metrics = await self.call_backend( self.model.tinker_calculate_metric, is_training=True, adapter_name=model_adapter) @@ -275,7 +278,10 @@ async def _do_save(): save_dir = checkpoint_manager.get_save_dir(model_id=body.model_id, is_sampler=False) await self.call_backend( self.model.save, - name=checkpoint_name, output_dir=save_dir, adapter_name=model_adapter, save_optimizer=True) + name=checkpoint_name, + output_dir=save_dir, + adapter_name=model_adapter, + save_optimizer=True) tinker_path = checkpoint_manager.save(body.model_id, name=checkpoint_name, is_sampler=False) return types.SaveWeightsResponse(path=tinker_path, type='save_weights') except Exception: diff --git a/src/twinkle/server/model/twinkle_handlers.py b/src/twinkle/server/model/twinkle_handlers.py index b780b279..01eadbe6 100644 --- a/src/twinkle/server/model/twinkle_handlers.py +++ b/src/twinkle/server/model/twinkle_handlers.py @@ -110,7 +110,9 @@ async def _task(): inputs = _parse_inputs(body.inputs) ret = await self.call_backend( self.model.forward, - inputs=inputs, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + inputs=inputs, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **extra_kwargs) return {'result': ret} inputs_list = body.inputs if isinstance(body.inputs, list) else [body.inputs] @@ -197,7 +199,9 @@ async def _task(): inputs = _parse_inputs(body.inputs) ret = await self.call_backend( self.model.forward_only, - inputs=inputs, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + inputs=inputs, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **extra_kwargs) return {'result': ret} inputs_list = body.inputs if isinstance(body.inputs, list) else [body.inputs] @@ -306,7 +310,9 @@ async def _task(): inputs[key] = torch.tensor(inputs[key]) ret = await self.call_backend( self.model.forward_backward, - inputs=all_inputs, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + inputs=all_inputs, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **extra_kwargs) return {'result': ret} inputs_list = body.inputs if isinstance(body.inputs, list) else [body.inputs] @@ -451,7 +457,8 @@ async def _task(): extra_kwargs = body.model_extra or {} ret = await self.call_backend( self.model.get_train_configs, - adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + adapter_name=self.resolve_model_adapter_name(adapter_name), + **extra_kwargs) return {'result': ret} return await run_task( @@ -467,7 +474,9 @@ async def _task(): extra_kwargs = body.model_extra or {} await self.call_backend( self.model.set_loss, - body.loss_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + body.loss_cls, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **extra_kwargs) await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_loss')) @@ -485,7 +494,9 @@ async def _task(): extra_kwargs = body.model_extra or {} await self.call_backend( self.model.set_optimizer, - body.optimizer_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + body.optimizer_cls, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **extra_kwargs) await run_task( self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_optimizer')) @@ -504,7 +515,9 @@ async def _task(): extra_kwargs = body.model_extra or {} await self.call_backend( self.model.set_lr_scheduler, - body.scheduler_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + body.scheduler_cls, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **extra_kwargs) await run_task( self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_lr_scheduler')) @@ -727,7 +740,9 @@ async def _task(): patch_cls = deserialize_object(body.patch_cls) await self.call_backend( self.model.apply_patch, - patch_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + patch_cls, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **extra_kwargs) await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='apply_patch')) @@ -767,7 +782,9 @@ async def _task(): extra_kwargs = body.model_extra or {} await self.call_backend( self.model.set_template, - body.template_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + body.template_cls, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **extra_kwargs) await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_template')) @@ -785,7 +802,9 @@ async def _task(): extra_kwargs = body.model_extra or {} await self.call_backend( self.model.set_processor, - body.processor_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + body.processor_cls, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **extra_kwargs) await run_task( self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_processor')) diff --git a/src/twinkle/server/sampler/twinkle_handlers.py b/src/twinkle/server/sampler/twinkle_handlers.py index 19100adb..22515b4e 100644 --- a/src/twinkle/server/sampler/twinkle_handlers.py +++ b/src/twinkle/server/sampler/twinkle_handlers.py @@ -493,15 +493,19 @@ async def _stream_generator(): while True: remaining = total_timeout - (loop.time() - start) if remaining <= 0: - yield json.dumps( - {'error': 'sample_stream exceeded the execution time bound', 'category': 'Server'}) + '\n' + yield json.dumps({ + 'error': 'sample_stream exceeded the execution time bound', + 'category': 'Server' + }) + '\n' break try: item = await asyncio.wait_for( loop.run_in_executor(None, q.get), timeout=min(single_get_timeout, remaining)) except asyncio.TimeoutError: - yield json.dumps( - {'error': 'sample_stream timed out waiting for the next token', 'category': 'Server'}) + '\n' + yield json.dumps({ + 'error': 'sample_stream timed out waiting for the next token', + 'category': 'Server' + }) + '\n' break if item == STREAM_SENTINEL: break diff --git a/src/twinkle/server/utils/task_errors.py b/src/twinkle/server/utils/task_errors.py index b0fa46eb..7e977cf1 100644 --- a/src/twinkle/server/utils/task_errors.py +++ b/src/twinkle/server/utils/task_errors.py @@ -7,7 +7,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Any, Optional +from typing import Any from twinkle_client.types.errors import ErrorCategory, ErrorPayload @@ -30,14 +30,14 @@ def task_error_payload( request_id: str, error_code: int = 500, category: ErrorCategory = ErrorCategory.Server, - traceback_text: Optional[str] = None, + traceback_text: str | None = None, ) -> dict[str, Any]: """Build an ``ErrorPayload`` and return it as a JSON-safe dict for storage. Traceback splitting and length trimming happen here so over-long text is never written to State_Backend. A ``User`` category carries no traceback (R5#6). """ - tb: Optional[str] = None + tb: str | None = None if category != ErrorCategory.User and traceback_text: tb = _trim_traceback(traceback_text) payload = ErrorPayload( diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/utils/task_queue/mixin.py index 9887e259..4260b893 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/utils/task_queue/mixin.py @@ -108,8 +108,7 @@ def _init_task_queue( # would let one leak block the whole queue forever. self._backend_executor = ThreadPoolExecutor(thread_name_prefix='twinkle-backend') # per-replica Admission_Gate; opt-in per deployment. - self._backend_admission: asyncio.Semaphore | None = ( - asyncio.Semaphore(1) if enable_admission_gate else None) + self._backend_admission: asyncio.Semaphore | None = (asyncio.Semaphore(1) if enable_admission_gate else None) self._event_loop: asyncio.AbstractEventLoop | None = None @@ -144,9 +143,8 @@ async def call_backend(self, fn: Callable[..., Any], /, *args: Any, admit: bool # check-then-acquire is race-free for a Semaphore(1): if not locked here, # acquire succeeds synchronously. if sem.locked(): - raise BackendBusyError( - 'This replica is waiting for a timed-out backend call to exit; ' - 'refusing to queue behind it.') + raise BackendBusyError('This replica is waiting for a timed-out backend call to exit; ' + 'refusing to queue behind it.') await sem.acquire() def _work() -> Any: diff --git a/src/twinkle/server/utils/task_queue/worker.py b/src/twinkle/server/utils/task_queue/worker.py index 1d951edb..121d7345 100644 --- a/src/twinkle/server/utils/task_queue/worker.py +++ b/src/twinkle/server/utils/task_queue/worker.py @@ -53,7 +53,7 @@ def __init__( config: TaskQueueConfig, task_metrics: TaskMetrics | None, deployment_name: str, - on_backend_timeout: 'Callable[[], Any] | None' = None, + on_backend_timeout: Callable[[], Any] | None = None, ) -> None: self._state = state self._config = config diff --git a/src/twinkle/utils/nccl_safe.py b/src/twinkle/utils/nccl_safe.py index d6633355..214f3a17 100644 --- a/src/twinkle/utils/nccl_safe.py +++ b/src/twinkle/utils/nccl_safe.py @@ -58,8 +58,8 @@ def wrapper(self, *args, **kwargs): tb = traceback.format_exc() if len(tb) > _TRACEBACK_LIMIT: tb = tb[-_TRACEBACK_LIMIT:] - logger.error('[nccl_safe_megatron] %s in %s on global rank %s:\n%s', type(exc).__name__, func.__name__, - _global_rank(), tb) + logger.error('[nccl_safe_megatron] %s in %s on global rank %s:\n%s', + type(exc).__name__, func.__name__, _global_rank(), tb) raise return wrapper diff --git a/tests/server/README.md b/tests/server/README.md new file mode 100644 index 00000000..44a998a8 --- /dev/null +++ b/tests/server/README.md @@ -0,0 +1,23 @@ +# Server test notes + +## Mock backend evidence boundary (spec T8.4 / R9#10) + +`server/model/backends/mock_model.py` is a stand-in backend for tests that must run +without a GPU or a real Ray-distributed model. Two properties bound what a test using +it can prove: + +1. **Every method takes `**kwargs` and performs no argument validation.** A test that + drives the mock backend therefore **cannot** be used as evidence for request/argument + validation behavior — the mock accepts anything. +2. **It never enters a real collective.** The mock does no NCCL communication, so a test + using it **cannot** be used as evidence for NCCL behavior (asymmetric failure, + collective mis-pairing, ReduceScatter, etc.). + +What the mock backend *can* evidence is exactly the parts that do not depend on the +backend's internals: the **dispatch path** (that a call reaches the backend via +`call_backend` / the task queue) and the **timeout / admission mechanisms** themselves +(that a slow or leaked call is bounded and the event loop stays responsive). + +Tests that need to prove validation or NCCL behavior are the GPU-gated end-to-end tests +under `tests/server/integration/test_nccl_safe_*_e2e.py` (run only with +`TWINKLE_TEST_GPU_E2E=1` against a real server). diff --git a/tests/server/contract/test_client_api_contract.py b/tests/server/contract/test_client_api_contract.py new file mode 100644 index 00000000..afed5e2c --- /dev/null +++ b/tests/server/contract/test_client_api_contract.py @@ -0,0 +1,42 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Zero-wire-change contract guard (T8.1 / R8 / Property 10). + +Re-exports the OpenAPI surface of all four apps and compares it field-by-field with +the baseline captured before this spec's implementation (T0.1). The diff must be +empty. Also asserts the load-bearing invariants: ``schedule_task_and_wait`` still +exists and the only client-side additions are ``types/base.py`` and ``types/errors.py``. +""" +from __future__ import annotations + +import pytest + +from tests.server.contract.client_api_harness import extract_full_surface, load_baseline + + +def test_openapi_surface_matches_baseline(): + current = extract_full_surface() + baseline = load_baseline() + assert current == baseline, ( + 'Client-facing OpenAPI surface changed vs the pre-implementation baseline; ' + 'this spec must be zero-wire-change. Diffing apps: ' + f'{[a for a in set(current) | set(baseline) if current.get(a) != baseline.get(a)]}') + + +def test_schedule_task_and_wait_not_removed(): + from twinkle.server.utils.task_queue.mixin import TaskQueueMixin + assert hasattr(TaskQueueMixin, 'schedule_task_and_wait') + + +def test_new_client_types_importable(): + # The only permitted client-side additions. + import twinkle_client.types.base as base + import twinkle_client.types.errors as errors + + for symbol in ('StrictRequest', 'ResponseModel', 'DataModel', 'backend_only'): + assert hasattr(base, symbol) + for symbol in ('ErrorPayload', 'ErrorCategory', 'QueueStateLiteral'): + assert hasattr(errors, symbol) + + +if __name__ == '__main__': + raise SystemExit(pytest.main([__file__, '-v'])) From d54d9a1f08eda8f3f9ab173d6b2a17af9bf78acb Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Tue, 15 Sep 2026 14:22:27 +0800 Subject: [PATCH 10/32] fix(state): store state timestamps as UTC so expiry is timezone-independent Root cause: _now_iso() wrote naive local time while _parse_timestamp() reads a naive ISO string as UTC; compared against a time.time()-based cutoff this skewed every expiry check by the host's UTC offset (premature deletion west of UTC, over-retention east; invisible on UTC/CI hosts). Fix (option A): _now_iso() now emits UTC-aware ISO; future_manager reuses it for record writes and its cleanup 'now' reverts to time.time() (both UTC epoch). _parse_timestamp is unchanged and still parses legacy naive records as UTC (same as before). Not a wire-schema change (R8 unaffected). Adds a timezone-independent regression test asserting a freshly written record parses to within 1s of time.time(); the suite passes under TZ=America/Los_Angeles. --- src/twinkle/server/state/future_manager.py | 11 +++----- src/twinkle/server/state/models.py | 7 +++-- tests/server/state/test_future_lifecycle.py | 30 ++++++++++++++------- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/twinkle/server/state/future_manager.py b/src/twinkle/server/state/future_manager.py index 31f5cac3..9d3b3a3c 100644 --- a/src/twinkle/server/state/future_manager.py +++ b/src/twinkle/server/state/future_manager.py @@ -2,14 +2,14 @@ from __future__ import annotations import functools -from datetime import datetime +import time from typing import Any from twinkle.server.utils.task_errors import task_error_payload from twinkle.utils.logger import get_logger from .backend.base import StateBackend from .base import BaseManager -from .models import FutureRecord +from .models import FutureRecord, _now_iso logger = get_logger() @@ -112,7 +112,7 @@ async def store_status( if result is not None and hasattr(result, 'model_dump'): result = result.model_dump() - now = datetime.now().isoformat() + now = _now_iso() await self._backend.update_atomic( self._make_key(request_id), functools.partial( @@ -162,10 +162,7 @@ async def cleanup_expired( counted here; they are removed on a later pass once terminal). """ all_records = await self.get_all() - # Use the same clock convention as the stored timestamps (_parse_timestamp on - # an ISO string) so the age computation is not skewed by _now_iso writing - # local time while _parse_timestamp reads naive ISO as UTC. - now = self._parse_timestamp(datetime.now().isoformat()) + now = time.time() expired_ids: list[str] = [] for request_id, record in all_records.items(): if record.status in _TERMINAL_STATUSES: diff --git a/src/twinkle/server/state/models.py b/src/twinkle/server/state/models.py index 60970022..d3d3133b 100644 --- a/src/twinkle/server/state/models.py +++ b/src/twinkle/server/state/models.py @@ -2,13 +2,16 @@ from __future__ import annotations import time -from datetime import datetime +from datetime import datetime, timezone from pydantic import BaseModel, Field from typing import Any def _now_iso() -> str: - return datetime.now().isoformat() + # UTC-aware so _parse_timestamp (which reads timestamps back as UTC) agrees with + # it and with time.time(); a naive local string would be misread as UTC and skew + # every expiry comparison by the host's UTC offset. + return datetime.now(timezone.utc).isoformat() class SessionRecord(BaseModel): diff --git a/tests/server/state/test_future_lifecycle.py b/tests/server/state/test_future_lifecycle.py index bf687044..c32a6172 100644 --- a/tests/server/state/test_future_lifecycle.py +++ b/tests/server/state/test_future_lifecycle.py @@ -5,7 +5,7 @@ """ from __future__ import annotations -from datetime import datetime +import time from unittest import mock import pytest @@ -20,11 +20,6 @@ def manager(tmp_path): return FutureManager(backend, expiration_timeout=300.0) -def _clock(manager) -> float: - """Now under the same convention the stored ISO timestamps use.""" - return manager._parse_timestamp(datetime.now().isoformat()) - - async def _store(manager, request_id, status, *, replica_id=None): await manager.store_status(request_id, status, model_id='m1', replica_id=replica_id) @@ -33,7 +28,7 @@ async def _store(manager, request_id, status, *, replica_id=None): async def test_non_terminal_with_live_replica_is_kept(manager): await _store(manager, 'r1', 'running', replica_id='replica-A') removed = await manager.cleanup_expired( - cutoff_time=_clock(manager) + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) + cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) assert removed == 0 rec = await manager.get('r1') assert rec is not None and rec.status == 'running' @@ -42,7 +37,7 @@ async def test_non_terminal_with_live_replica_is_kept(manager): @pytest.mark.asyncio async def test_non_terminal_orphan_is_failed_not_deleted(manager): await _store(manager, 'r2', 'running', replica_id='dead-replica') - await manager.cleanup_expired(cutoff_time=_clock(manager) + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) + await manager.cleanup_expired(cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) rec = await manager.get('r2') assert rec is not None # NOT deleted (Property 6) assert rec.status == 'failed' @@ -53,7 +48,7 @@ async def test_non_terminal_orphan_is_failed_not_deleted(manager): async def test_non_terminal_over_absolute_ttl_is_failed(manager): await _store(manager, 'r3', 'running', replica_id='replica-A') # absolute_ttl=0 makes any positive age exceed the bound. - await manager.cleanup_expired(cutoff_time=_clock(manager) + 10, alive_replica_ids={'replica-A'}, absolute_ttl=0.0) + await manager.cleanup_expired(cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}, absolute_ttl=0.0) rec = await manager.get('r3') assert rec is not None and rec.status == 'failed' @@ -62,7 +57,7 @@ async def test_non_terminal_over_absolute_ttl_is_failed(manager): async def test_terminal_expired_is_deleted(manager): await _store(manager, 'r4', 'completed', replica_id='replica-A') removed = await manager.cleanup_expired( - cutoff_time=_clock(manager) + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) + cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) assert removed == 1 assert await manager.get('r4') is None @@ -93,3 +88,18 @@ async def test_replica_id_set_at_creation_not_overwritten(manager): await manager.store_status('r7', 'running', model_id='m1', replica_id='replica-B') rec = await manager.get('r7') assert rec.replica_id == 'replica-A' # creation value preserved + + +@pytest.mark.asyncio +async def test_stored_timestamps_align_with_wall_clock_regardless_of_host_tz(manager): + """Writer (_now_iso), reader (_parse_timestamp) and time.time() must agree. + + A record written now must parse to within a second of time.time() on any host, + not skewed by the host's UTC offset (the former naive-local / read-as-UTC bug). + """ + before = time.time() + await _store(manager, 'r8', 'running', replica_id='replica-A') + after = time.time() + rec = await manager.get('r8') + parsed = manager._parse_timestamp(rec.created_at) + assert before - 1 <= parsed <= after + 1 From f97810482ad04a7e0b7646701089c33477665305 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Tue, 15 Sep 2026 15:33:46 +0800 Subject: [PATCH 11/32] fix: restore multi lora state after load failure --- src/twinkle/model/multi_lora.py | 14 ++++--- .../test_multi_lora_target_parameters.py | 39 ++++++++++++++++++- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/src/twinkle/model/multi_lora.py b/src/twinkle/model/multi_lora.py index ff776610..6eb80ed2 100644 --- a/src/twinkle/model/multi_lora.py +++ b/src/twinkle/model/multi_lora.py @@ -193,12 +193,14 @@ def _after(_module): _before(_module) else: _before(self.module) - yield adapter_name - if isinstance(self.module, list): - for _module in self.module: - _after(_module) - else: - _after(self.module) + try: + yield adapter_name + finally: + if isinstance(self.module, list): + for _module in self.module: + _after(_module) + else: + _after(self.module) # self.deactivate_adapter() def check_length( diff --git a/tests/model/test_multi_lora_target_parameters.py b/tests/model/test_multi_lora_target_parameters.py index b28ef6b3..c76854ce 100644 --- a/tests/model/test_multi_lora_target_parameters.py +++ b/tests/model/test_multi_lora_target_parameters.py @@ -2,6 +2,7 @@ import sys import types +import pytest import torch from peft import LoraConfig, get_peft_model from peft.utils import set_peft_model_state_dict @@ -45,6 +46,42 @@ def forward(self, x, expert_idx=0): return self.mlp.experts(x, expert_idx=expert_idx) +class FakePeftModule: + + def __init__(self, peft_config, active_adapter): + self.peft_config = peft_config + self.active_adapter = active_adapter + + +def test_save_context_restores_peft_state_when_load_fails(): + from twinkle.model.multi_lora import LoraTenant, MultiLora + + original_config = {'lora_0': object(), 'lora_1': object()} + modules = [ + FakePeftModule(original_config, 'lora_1'), + FakePeftModule(original_config, 'lora_0'), + ] + multi_lora = MultiLora(max_loras=2, max_r=4) + multi_lora.module = modules + multi_lora.loras = [ + LoraTenant( + index=0, + adapter_name='lora_0', + config=_make_target_cfg(), + tenant_adapter_name='tenant', + tenant_config=_make_target_cfg(), + ) + ] + + with pytest.raises(RuntimeError, match='load failed'): + with multi_lora.save_context('tenant') as adapter_name: + assert adapter_name == 'lora_0' + raise RuntimeError('load failed') + + assert [module.peft_config for module in modules] == [original_config, original_config] + assert [module.active_adapter for module in modules] == ['lora_1', 'lora_0'] + + def test_peft_target_parameter_key_shapes_for_3d_experts(): model = FakeModel() cfg = LoraConfig( @@ -266,4 +303,4 @@ def test_multilora_transformers_installs_target_parameters_once(): assert test_target_parameter_multi_lora_updates_only_active_adapter() == True assert test_multilora_releases_target_parameter_slot_to_initial_weights() == True assert test_multilora_state_dict_round_trips_target_parameters() == True - assert test_multilora_transformers_installs_target_parameters_once() == True \ No newline at end of file + assert test_multilora_transformers_installs_target_parameters_once() == True From 26a4311d227d01b1c977c38ba4daeda59174e04e Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Tue, 15 Sep 2026 21:31:03 +0800 Subject: [PATCH 12/32] feat(server): finalize execution guarantees (backend boundary + stream/gen bounds) Complete the server-execution-guarantees spec by routing the remaining sampler paths through the Blocking_Call_Boundary and bounding every long-lived operation: - sampler: submit/collect/cancel generation, unload, and streaming now go through call_backend; _await_generation is bounded by the effective execution timeout and _stream_queue enforces a double timeout (total + per-token) instead of blocking indefinitely. - task queue: thread collect_width through _init_task_queue so state hygiene computes the absolute survival TTL without a separate set_execution_bounds hop. - errors: stream/generation failures surface as ErrorPayload wire frames rather than silent drops. - deps: bump tinker to 0.29.0 (python>=3.11) and refresh poetry.lock. - tests/docs: add error-wire, stream-guarantee and nccl_safe coverage; refresh the client API contract baseline (zero wire change). --- .../Usage Guide/Server and Client/Server.md | 32 +- .../\346\234\215\345\212\241\347\253\257.md" | 10 +- poetry.lock | 2933 +++++- pyproject.toml | 2 +- src/twinkle/infra/__init__.py | 5 +- src/twinkle/server/gateway/tinker_handlers.py | 5 +- src/twinkle/server/model/app.py | 52 +- .../server/model/backends/megatron_model.py | 3 +- .../server/model/backends/mock_model.py | 2 +- .../model/backends/transformers_model.py | 3 +- src/twinkle/server/model/tinker_handlers.py | 58 +- src/twinkle/server/model/twinkle_handlers.py | 14 +- .../server/processor/twinkle_handlers.py | 3 +- src/twinkle/server/sampler/app.py | 33 +- src/twinkle/server/sampler/tinker_handlers.py | 33 +- .../server/sampler/twinkle_handlers.py | 137 +- src/twinkle/server/state/future_manager.py | 46 +- src/twinkle/server/state/model_manager.py | 4 +- src/twinkle/server/state/models.py | 5 +- src/twinkle/server/state/replica_registry.py | 15 +- src/twinkle/server/state/server_state.py | 39 +- src/twinkle/server/utils/task_errors.py | 24 +- .../server/utils/task_queue/__init__.py | 3 +- src/twinkle/server/utils/task_queue/config.py | 10 +- src/twinkle/server/utils/task_queue/mixin.py | 151 +- src/twinkle/server/utils/task_queue/types.py | 6 +- src/twinkle/server/utils/task_queue/worker.py | 17 +- src/twinkle/utils/nccl_safe.py | 10 +- src/twinkle_client/types/errors.py | 27 +- tests/infra/test_ray_get_timeout.py | 27 +- tests/server/README.md | 6 +- .../server/contract/client_api_baseline.json | 8148 +++++++++++++++-- tests/server/contract/client_api_harness.py | 138 +- .../contract/test_client_api_contract.py | 11 +- tests/server/contract/test_error_wire.py | 39 + .../server/integration/test_actor_recovery.py | 17 + .../integration/test_blocking_boundary.py | 121 +- .../integration/test_nccl_safe_tinker_e2e.py | 30 +- .../integration/test_nccl_safe_twinkle_e2e.py | 8 +- tests/server/model/test_replica_lifecycle.py | 11 +- tests/server/model/test_tinker_handlers.py | 4 +- .../server/sampler/test_stream_guarantees.py | 113 + tests/server/sampler/test_tinker_handlers.py | 5 + tests/server/state/test_error_payload.py | 38 +- tests/server/state/test_future_lifecycle.py | 45 +- tests/server/state/test_managers.py | 6 + .../server/static/backend_call_exemptions.py | 2 +- .../static/test_no_direct_backend_call.py | 74 +- tests/server/utils/task_queue/test_config.py | 13 + tests/server/utils/test_task_errors.py | 10 +- tests/server/utils/test_task_queue_mixin.py | 30 +- tests/twinkle_client/test_types_contract.py | 43 +- tests/utils/test_nccl_safe.py | 18 + 53 files changed, 11305 insertions(+), 1334 deletions(-) create mode 100644 tests/server/contract/test_error_wire.py create mode 100644 tests/server/sampler/test_stream_guarantees.py create mode 100644 tests/utils/test_nccl_safe.py diff --git a/docs/source_en/Usage Guide/Server and Client/Server.md b/docs/source_en/Usage Guide/Server and Client/Server.md index d57084f1..5e67e37c 100644 --- a/docs/source_en/Usage Guide/Server and Client/Server.md +++ b/docs/source_en/Usage Guide/Server and Client/Server.md @@ -453,30 +453,30 @@ Additionally, this refactor introduces two new top-level fields — `telemetry` ## Execution time bounds -Every backend call has a configuration-computable time bound. The single source of -the bound is the *effective execution timeout* `T`: it equals `execution_timeout` -from the task-queue config, or `3600s` when `execution_timeout` is set to `0` -("no configured limit"; a startup warning is logged). `T` drives both the -`ray.get` timeout on the backend and the `asyncio.wait_for` around each task — -there is no second, independently tunable timeout. The default `execution_timeout` -is `1800s`. +Every backend call has a finite time bound. `T` is the effective task execution +timeout: it equals `execution_timeout`, or `3600s` when that setting is `0`. +`asyncio.wait_for` uses `T`. The Ray wait uses `R`, which is a method's explicit +constant timeout when present and otherwise `T`. The default `T` is `1800s`. Two distinct bounds follow, and they must not be collapsed into one number: | Bound | Expression | Meaning | |-------|------------|---------| | Record-terminal bound | `queue_timeout + T` | After this, a task's future record is guaranteed to be in a terminal state (`completed`/`failed`). Use it for alerting thresholds and client polling total-timeout. | -| Resource-release bound | `Collect_Width × T` | After this, the executor thread and the in-flight model-actor call for that task are guaranteed to have finished. Use it for capacity planning. | +| Resource-release bound | `Collect_Width × R` from execution start, or `queue_timeout + Collect_Width × R` from submission | After this, the executor thread and the in-flight model-actor call for that task are guaranteed to have finished. Use it for capacity planning. | `Collect_Width = len(self._actors) = world_size = tp × pp × dp` — the number of futures each `remote_function` collection waits on per call. Evidence: `LazyCollect._get_result` iterates `self._futures`, which come from `_get_workers(self._actors, execute)` (`infra/__init__.py`), covering every actor — -not just the data-parallel width. On a `tp=8` deployment the resource-release bound -is therefore `8 × T`, not `T`. - -The **difference** between the two bounds (`Collect_Width × T − (queue_timeout + T)`) -is the longest time the per-replica Admission_Gate can stay closed after a timeout: -the record is already terminal, but a leaked executor thread may still hold the gate -until its `ray.get` returns or raises. During that window newly arriving tasks fail -fast with a `Server`/503 error rather than queueing behind the stuck call. +not just the data-parallel width. On a `tp=8` deployment the execution-start +resource-release bound is therefore `8 × R`, not `R`. + +After the task record becomes terminal, the per-replica Admission_Gate can remain +closed for at most `max(0, Collect_Width × R − T)`: the record is already terminal, +but a leaked executor thread may still hold the gate until its `ray.get` returns or +raises. During that window newly arriving tasks fail fast with a `server`/503 error. + +Each persisted future stores its immutable `absolute_deadline` when it is created. +Cleanup therefore reaches the same decision regardless of which deployment process +holds the cleanup lease. diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" index 0794fb08..db71e41a 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" @@ -453,15 +453,17 @@ twinkle-server check-config -c server_config.yaml ## 执行时间上界 -每一次 backend 调用都存在一个可由配置计算的时间上界。上界的唯一来源是 **有效 execution timeout** `T`：它等于 task-queue 配置中的 `execution_timeout`；当 `execution_timeout` 配置为 `0`（表示「无配置上限」）时取 `3600` 秒，并在启动日志输出一条 warning。`T` 同时作用于 backend 的 `ray.get` 超时与每个任务外围的 `asyncio.wait_for`——不存在第二个可独立调节的超时值。`execution_timeout` 的默认值为 `1800` 秒。 +每一次 backend 调用都存在有限时间上界。`T` 是任务的有效 execution timeout：等于 task-queue 配置中的 `execution_timeout`；配置为 `0` 时取 `3600` 秒。`asyncio.wait_for` 使用 `T`。Ray 等待使用 `R`：方法显式声明 timeout 时取该常量，否则取 `T`。`T` 的默认值为 `1800` 秒。 由此派生出两个**不同**的上界，二者不得合成一个数： | 上界 | 表达式 | 含义 | |------|--------|------| | 记录终态上界 | `queue_timeout + T` | 超过它后，任务的 future 记录必处于终态（`completed`/`failed`）。用于设置告警阈值与客户端轮询总超时。 | -| 资源释放上界 | `Collect_Width × T` | 超过它后，该任务占用的 executor 线程与 model actor 在飞调用必已结束。用于容量规划。 | +| 资源释放上界 | 从执行开始为 `Collect_Width × R`；从提交开始为 `queue_timeout + Collect_Width × R` | 超过它后，该任务占用的 executor 线程与 model actor 在飞调用必已结束。用于容量规划。 | -`Collect_Width = len(self._actors) = world_size = tp × pp × dp`——即每次 `remote_function` 结果收集所等待的 future 个数。证据：`LazyCollect._get_result` 遍历的 `self._futures` 来自 `_get_workers(self._actors, execute)`（`infra/__init__.py`），覆盖全部 actor，而非 data-parallel 宽度。因此在 `tp=8` 的部署上，资源释放上界是 `8 × T` 而非 `T`。 +`Collect_Width = len(self._actors) = world_size = tp × pp × dp`——即每次 `remote_function` 结果收集所等待的 future 个数。证据：`LazyCollect._get_result` 遍历的 `self._futures` 来自 `_get_workers(self._actors, execute)`（`infra/__init__.py`），覆盖全部 actor，而非 data-parallel 宽度。因此在 `tp=8` 的部署上，从执行开始的资源释放上界是 `8 × R` 而非 `R`。 -两个上界的**差值**（`Collect_Width × T − (queue_timeout + T)`）即 per-replica 准入闸门在一次超时后需保持关闭的最长时长：此时记录已是终态，但泄漏的 executor 线程可能仍持有闸门，直到其 `ray.get` 返回或抛出。在该窗口内新到达的任务会以 `Server`/503 错误快速失败，而不是排队等待卡住的调用。 +任务记录进入终态后，per-replica 准入闸门额外保持关闭的最长时长为 `max(0, Collect_Width × R − T)`：此时记录已是终态，但泄漏的 executor 线程可能仍持有闸门，直到其 `ray.get` 返回或抛出。在该窗口内新到达的任务会以 `server`/503 错误快速失败。 + +每条持久化 future 在创建时写入不可变的 `absolute_deadline`，因此无论哪个 deployment 进程持有 cleanup lease，清理结果都由任务自身契约决定。 diff --git a/poetry.lock b/poetry.lock index 773c9b14..4a7b37e2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,17 @@ -# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. + +[[package]] +name = "absl-py" +version = "2.5.0" +description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, + {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, +] [[package]] name = "accelerate" @@ -210,6 +223,7 @@ files = [ [package.dependencies] aiohappyeyeballs = ">=2.5.0" aiosignal = ">=1.4.0" +async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" @@ -226,7 +240,7 @@ description = "CORS support for aiohttp" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "aiohttp_cors-0.8.1-py3-none-any.whl", hash = "sha256:3180cf304c5c712d626b9162b195b1db7ddf976a2a25172b35bb2448b890a80d"}, {file = "aiohttp_cors-0.8.1.tar.gz", hash = "sha256:ccacf9cb84b64939ea15f859a146af1f662a6b1d68175754a07315e305fb1403"}, @@ -268,9 +282,10 @@ files = [ name = "aliyun-python-sdk-core" version = "2.11.5" description = "The core module of Aliyun Python SDK." -optional = false +optional = true python-versions = "*" groups = ["main"] +markers = "extra == \"eval\"" files = [ {file = "aliyun-python-sdk-core-2.11.5.tar.gz", hash = "sha256:577265c630c02207c692ca19958bd21665d56208306a834d0885e7770553975e"}, ] @@ -282,9 +297,10 @@ pycryptodome = ">=3.4.7" name = "aliyun-python-sdk-kms" version = "2.16.5" description = "The kms module of Aliyun Python sdk." -optional = false +optional = true python-versions = "*" groups = ["main"] +markers = "extra == \"eval\"" files = [ {file = "aliyun-python-sdk-kms-2.16.5.tar.gz", hash = "sha256:f328a8a19d83ecbb965ffce0ec1e9930755216d104638cd95ecd362753b813b3"}, {file = "aliyun_python_sdk_kms-2.16.5-py2.py3-none-any.whl", hash = "sha256:24b6cdc4fd161d2942619479c8d050c63ea9cd22b044fe33b60bbb60153786f0"}, @@ -324,7 +340,7 @@ description = "The official Python library for the anthropic API" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "anthropic-0.84.0-py3-none-any.whl", hash = "sha256:861c4c50f91ca45f942e091d83b60530ad6d4f98733bfe648065364da05d29e7"}, {file = "anthropic-0.84.0.tar.gz", hash = "sha256:72f5f90e5aebe62dca316cb013629cfa24996b0f5a4593b8c3d712bc03c43c37"}, @@ -370,6 +386,7 @@ files = [ ] [package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} @@ -383,7 +400,7 @@ description = "tvm ffi" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "apache_tvm_ffi-0.1.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d911cbbc83bf12a0d9ec03e5315ff1bb92d95702fe912cd7a050393274382e71"}, {file = "apache_tvm_ffi-0.1.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1c8dd4018420c0d14bace688594710909ce198056ff8ac2ad1cd462b30fe1bdd"}, @@ -437,12 +454,25 @@ description = "Read/rewrite/write Python ASTs" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5"}, {file = "astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e"}, ] +[[package]] +name = "async-timeout" +version = "5.0.1" +description = "Timeout context manager for asyncio programs" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "python_full_version < \"3.11.3\" and extra == \"server\" or python_version == \"3.10\"" +files = [ + {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, + {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, +] + [[package]] name = "attrs" version = "25.4.0" @@ -471,6 +501,19 @@ files = [ [package.extras] dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] +[[package]] +name = "backports-asyncio-runner" +version = "1.2.0" +description = "Backport of asyncio.Runner, a context manager that controls event loop life cycle." +optional = true +python-versions = "<3.11,>=3.8" +groups = ["main"] +markers = "python_version == \"3.10\" and extra == \"test\"" +files = [ + {file = "backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5"}, + {file = "backports_asyncio_runner-1.2.0.tar.gz", hash = "sha256:a5aa7b2b7d8f8bfcaa2b57313f70792df84e32a2a746f585213373f900b42162"}, +] + [[package]] name = "beautifulsoup4" version = "4.14.3" @@ -502,7 +545,7 @@ description = "Python bindings for the Rust blake3 crate" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "blake3-1.0.8-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:8956bb9aec47b6c37ccce935a943588f1f5e6e2e85d43bb7cb76a574238f8a9b"}, {file = "blake3-1.0.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7adbbee5dd0c302218eb8acdfd82b7006930eb5798f56f79f9cca89f6f192662"}, @@ -625,7 +668,7 @@ description = "Extensible memoizing collections and decorators" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "cachetools-7.0.1-py3-none-any.whl", hash = "sha256:8f086515c254d5664ae2146d14fc7f65c9a4bce75152eb247e5a9c5e6d7b2ecf"}, {file = "cachetools-7.0.1.tar.gz", hash = "sha256:e31e579d2c5b6e2944177a0397150d312888ddf4e16e12f1016068f0c03b8341"}, @@ -638,7 +681,7 @@ description = "CBOR (de)serializer with extensive tag support" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "cbor2-5.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2263c0c892194f10012ced24c322d025d9d7b11b41da1c357f3b3fe06676e6b7"}, {file = "cbor2-5.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ffe4ca079f6f8ed393f5c71a8de22651cb27bd50e74e2bcd6bc9c8f853a732b"}, @@ -702,10 +745,10 @@ files = [ name = "cffi" version = "2.0.0" description = "Foreign Function Interface for Python calling C code." -optional = true +optional = false python-versions = ">=3.9" groups = ["main"] -markers = "platform_python_implementation != \"PyPy\" and (extra == \"vllm\" or extra == \"ray\") or extra == \"vllm\" and implementation_name == \"pypy\"" +markers = "(platform_python_implementation != \"PyPy\" or implementation_name == \"pypy\") and (implementation_name == \"pypy\" or extra == \"rl\" or extra == \"async-rl\")" files = [ {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, @@ -923,10 +966,9 @@ files = [ name = "click" version = "8.3.1" description = "Composable command line interface toolkit" -optional = true +optional = false python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\" or extra == \"tinker\"" files = [ {file = "click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6"}, {file = "click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a"}, @@ -942,7 +984,7 @@ description = "Pickler class to extend the standard pickle.Pickler functionality optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a"}, {file = "cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414"}, @@ -955,7 +997,7 @@ description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" groups = ["main"] -markers = "(extra == \"vllm\" or extra == \"ray\" or extra == \"docs\") and sys_platform == \"win32\" or platform_system == \"Windows\"" +markers = "(platform_system == \"Windows\" or sys_platform == \"win32\" or extra == \"eval\") and (platform_system == \"Windows\" or extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"test\" or extra == \"docs\")" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -968,7 +1010,7 @@ description = "Terminal string styling done right, in Python." optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "colorful-0.5.8-py2.py3-none-any.whl", hash = "sha256:a9381fdda3337fbaba5771991020abc69676afa102646650b759927892875992"}, {file = "colorful-0.5.8.tar.gz", hash = "sha256:bb16502b198be2f1c42ba3c52c703d5f651d826076817185f0294c1a549a7445"}, @@ -977,6 +1019,25 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[[package]] +name = "colorlog" +version = "6.12.0" +description = "Add colours to the output of Python's logging module." +optional = true +python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "colorlog-6.12.0-py3-none-any.whl", hash = "sha256:30d392604e9110045a2c2aeefc27d7a017abbab63f3a8aee594eac0801df784e"}, + {file = "colorlog-6.12.0.tar.gz", hash = "sha256:2a7924c1dadf18b22a0eb8b06d1c7b01d5341707ec1641eb6fcc4fde0c3e8e5f"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} + +[package.extras] +development = ["black", "flake8", "mypy", "pytest", "types-colorama"] + [[package]] name = "commonmark" version = "0.9.1" @@ -1000,7 +1061,7 @@ description = "Library for utilization of compressed safetensors of neural netwo optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "compressed_tensors-0.13.0-py3-none-any.whl", hash = "sha256:3518799c9baf034eb642efb551db6b0537b8713d45a64fe4def26f7f8d6cabec"}, {file = "compressed_tensors-0.13.0.tar.gz", hash = "sha256:23893824d3498ea3f1a829f14a8fa85f9a5e76a34c711a038b8d7c619ca9a67c"}, @@ -1020,9 +1081,10 @@ dev = ["black (==22.12.0)", "flake8 (>=3.8.3)", "isort (==5.8.0)", "nbconvert (> name = "crcmod" version = "1.7" description = "CRC Generator" -optional = false +optional = true python-versions = "*" groups = ["main"] +markers = "extra == \"eval\"" files = [ {file = "crcmod-1.7.tar.gz", hash = "sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e"}, ] @@ -1034,7 +1096,7 @@ description = "cryptography is a package which provides cryptographic recipes an optional = true python-versions = "!=3.9.0,!=3.9.1,>=3.8" groups = ["main"] -markers = "extra == \"ray\" or extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad"}, {file = "cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b"}, @@ -1089,6 +1151,7 @@ files = [ [package.dependencies] cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} +typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11.0\""} [package.extras] docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] @@ -1107,7 +1170,7 @@ description = "Python bindings for CUDA" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "cuda_bindings-13.1.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4400370a83f1538e25ed4c18c34a0e9d5fad39741e282e69ce24d1479a11017d"}, {file = "cuda_bindings-13.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:81f92500e2f6aec2dac00a5a1ce77d5aa77ea77b606dc484d951f1f2cc3eaa13"}, @@ -1142,7 +1205,7 @@ description = "Pathfinder for CUDA components" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "cuda_pathfinder-1.4.0-py3-none-any.whl", hash = "sha256:437079ca59e7b61ae439ecc501d69ed87b3accc34d58153ef1e54815e2c2e118"}, ] @@ -1154,7 +1217,7 @@ description = "CUDA Python: Performance meets Productivity" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "cuda_python-13.1.1-py3-none-any.whl", hash = "sha256:944cc4fe6482673d28dd545797a28840945a1668739328fa2ad1e9be4f7050d9"}, ] @@ -1173,7 +1236,7 @@ description = "CuPy: NumPy & SciPy for GPU" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\" and sys_platform != \"darwin\"" +markers = "sys_platform != \"darwin\" and (extra == \"rl\" or extra == \"async-rl\")" files = [ {file = "cupy_cuda12x-14.0.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:b42da54c9da0d5a7748e4120f13c47594d3e1fc2741b712591aa915517741096"}, {file = "cupy_cuda12x-14.0.1-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:7c775e1e1ebc0c4c9f94a4c6bb66a0c07d109de5dfcef671f9e4056df4bd81ca"}, @@ -1244,6 +1307,19 @@ tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "aiohttp", "decorator", "elastics torch = ["torch"] vision = ["Pillow (>=9.4.0)"] +[[package]] +name = "defusedxml" +version = "0.7.1" +description = "XML bomb protection for Python stdlib modules" +optional = true +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, + {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, +] + [[package]] name = "depyf" version = "0.20.0" @@ -1251,7 +1327,7 @@ description = "Decompile python functions, from bytecode to source code!" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "depyf-0.20.0-py3-none-any.whl", hash = "sha256:d31effad4261cebecb58955d832e448ace88f432328f95f82fd99c30fd9308d4"}, {file = "depyf-0.20.0.tar.gz", hash = "sha256:fb7683bd72c44f67b56029df2c47721e9a02ffa4d7b19095f1c54c4ebf797a98"}, @@ -1287,7 +1363,7 @@ description = "Disk Cache -- Disk and file backed persistent cache." optional = true python-versions = ">=3" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19"}, {file = "diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc"}, @@ -1300,7 +1376,7 @@ description = "Distribution utilities" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, @@ -1313,7 +1389,7 @@ description = "Distro - an OS platform information API" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"tinker\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" files = [ {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, @@ -1326,7 +1402,7 @@ description = "DNS toolkit" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af"}, {file = "dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f"}, @@ -1348,7 +1424,7 @@ description = "Parse Python docstrings in reST, Google and Numpydoc format" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "docstring_parser-0.17.0-py3-none-any.whl", hash = "sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708"}, {file = "docstring_parser-0.17.0.tar.gz", hash = "sha256:583de4a309722b3315439bb31d64ba3eebada841f2e2cee23b99df001434c912"}, @@ -1372,6 +1448,103 @@ files = [ {file = "docutils-0.16.tar.gz", hash = "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"}, ] +[[package]] +name = "dotenv" +version = "0.9.9" +description = "Deprecated package" +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "dotenv-0.9.9-py2.py3-none-any.whl", hash = "sha256:29cf74a087b31dafdb5a446b6d7e11cbce8ed2741540e2339c69fbef92c94ce9"}, +] + +[package.dependencies] +python-dotenv = "*" + +[[package]] +name = "editdistance" +version = "0.8.1" +description = "Fast implementation of the edit distance (Levenshtein distance)" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "editdistance-0.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:adeb705f32b93accc74960d227875abff150ee42d676e428536361fe5f8f5388"}, + {file = "editdistance-0.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3de77951b105d0972deec7684a0b3d1a9dee69c9b5d34f6e2acc0d76cd4a1c52"}, + {file = "editdistance-0.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5e88efb052d45e924606c305cb833a80579dca3e8e4ff01309d50ba2c1c0bbd5"}, + {file = "editdistance-0.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0247e7a1e9c66ea75211a97e725366bff19a52aac2c838ed5f90025630e976dd"}, + {file = "editdistance-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67d143429a49ab552411505f550a0fb4285a1d4336e096804d233ec495ac20fc"}, + {file = "editdistance-0.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca9d3be2b10e5d44a950a4bd1e84bca9ebbecd364bce0cf5693bf8224c78eaef"}, + {file = "editdistance-0.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5c72aa1df8535f2e2b3d8773a1a7da091bc1a7e52bb396e7e48d375ba687e7b2"}, + {file = "editdistance-0.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9a606c34a2a6cc190e4fffc856b36333cdcf1f1fab5b22bd3088e585c22d6ca0"}, + {file = "editdistance-0.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5af173d442ffac33b7c7990132f97f88818a3abf4b21c0c702a7022df37c0c5c"}, + {file = "editdistance-0.8.1-cp310-cp310-win32.whl", hash = "sha256:fd64b58f5a7b59afd9d75982aaeeacd2a98498bf472fa0360c122ffe6ea4c871"}, + {file = "editdistance-0.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:6c7c62c3cae45ca1fa01bb2722b297b9de1e3a244ac44cfba88bdcb488fe6aee"}, + {file = "editdistance-0.8.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:486105603a273d73d12a54f347dffa70ab281749d7c3879658b377bc49e4b98c"}, + {file = "editdistance-0.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fad081f5f86a175c1a09a4e9e45b95c9349e454c21e181e842e01c85f1f536fc"}, + {file = "editdistance-0.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8cb78e125f6759398885a775f5eed07c2bb72b2f86da43e674c6b6a3335b273b"}, + {file = "editdistance-0.8.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3778ca60aa89def9144b70e330bcec5330c7da1d69cb28c612e90b84510a1d3d"}, + {file = "editdistance-0.8.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fba945eaa0436cf40bc53d7e299dc537c7c71353379a095b7459ff4af910da33"}, + {file = "editdistance-0.8.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:877f2a0d801f32bc1a1878901ffb947b974361e849c66e314a7f1d786a446b58"}, + {file = "editdistance-0.8.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e79d351ca40a6ead5f3763253fd7521572ee0d3e5d42538630e56d10f48db481"}, + {file = "editdistance-0.8.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:70ed382b3052a51161bad0149d4665003bf3b949fce0b01bf1253a4cc1a88239"}, + {file = "editdistance-0.8.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a529bfb384c4000775d76739c4e64f73337f0f5a3784933b1321b577a62bed4e"}, + {file = "editdistance-0.8.1-cp311-cp311-win32.whl", hash = "sha256:b082232429e731f181af7f7d2bcf79da6ca8fadd04e9086c11e2973f7d330c81"}, + {file = "editdistance-0.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:cef1a4359252a49f2c4718e64e9d40027d9d951b289d045bdb278656e59f6af8"}, + {file = "editdistance-0.8.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04af61b3fcdd287a07c15b6ae3b02af01c5e3e9c3aca76b8c1d13bd266b6f57"}, + {file = "editdistance-0.8.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:18fc8b6eaae01bfd9cf999af726c1e8dcf667d120e81aa7dbd515bea7427f62f"}, + {file = "editdistance-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a87839450a5987028738d061ffa5ef6a68bac2ddc68c9147a8aae9806629c7f"}, + {file = "editdistance-0.8.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24b5f9c9673c823d91b5973d0af8b39f883f414a55ade2b9d097138acd10f31e"}, + {file = "editdistance-0.8.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c59248eabfad603f0fba47b0c263d5dc728fb01c2b6b50fb6ca187cec547fdb3"}, + {file = "editdistance-0.8.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e239d88ff52821cf64023fabd06a1d9a07654f364b64bf1284577fd3a79d0e"}, + {file = "editdistance-0.8.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2f7f71698f83e8c83839ac0d876a0f4ef996c86c5460aebd26d85568d4afd0db"}, + {file = "editdistance-0.8.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:04e229d6f4ce0c12abc9f4cd4023a5b5fa9620226e0207b119c3c2778b036250"}, + {file = "editdistance-0.8.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e16721636da6d6b68a2c09eaced35a94f4a4a704ec09f45756d4fd5e128ed18d"}, + {file = "editdistance-0.8.1-cp312-cp312-win32.whl", hash = "sha256:87533cf2ebc3777088d991947274cd7e1014b9c861a8aa65257bcdc0ee492526"}, + {file = "editdistance-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:09f01ed51746d90178af7dd7ea4ebb41497ef19f53c7f327e864421743dffb0a"}, + {file = "editdistance-0.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0b6f52a9d7d434f6882db3dc0340e42da6f177644c23f6a02a739b6247a14b82"}, + {file = "editdistance-0.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d5b413aeb8fe3f77a9e95485fcb64ce84e6b14bcd368124d38bd0062167b3456"}, + {file = "editdistance-0.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4ba571e8b6796ad34faeb8581ddc311c35946b2fc183eaebfef59e12ea3538b3"}, + {file = "editdistance-0.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e361e932a85ee1f7091fcb8f7e4619681592c1a0cca251dfd26976dd58254"}, + {file = "editdistance-0.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf200104ed4923d4f51ca543bc8488732a31a17848058e65bcba855a7eee2bd2"}, + {file = "editdistance-0.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:387d3bb45befbf8514eb8d17180307580efe4ebaa40ad8b2c14eb04c52ad18a0"}, + {file = "editdistance-0.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8e4ea90e92f0e3494bdddad45928393094c258aeef9e4def81a39c3429df0e19"}, + {file = "editdistance-0.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5d746ecbf7db7fe0f93ba6971ac43225aac877818a3c2003d41436fee9b33905"}, + {file = "editdistance-0.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d789a8ef6fe7cb287ff199381bbd62e9fb2ba4b1e69db817b761d9b42bf8ca7b"}, + {file = "editdistance-0.8.1-cp38-cp38-win32.whl", hash = "sha256:dbbf050fece6c78838a8a95fa4e9a4132023c3d85138870ac83c9dc1dfbfe513"}, + {file = "editdistance-0.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:1c49df0717f64a2c8869edc32c01ba9a22ba20b6cc482876c1067e3a92a6cb2d"}, + {file = "editdistance-0.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4d8e9a3e65a68c13dcadc1d2caca620f1716a8d02f2602047e0721b509161ec7"}, + {file = "editdistance-0.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7994a6a0a6ae92db87c144e12f1549ca0e50f43c6cc64e32c628e7af6b9c74b6"}, + {file = "editdistance-0.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dbe0cbc15466e9b7fbf73e34bdcae11cb0c2acd09a60ef4740f2172f9aa5e751"}, + {file = "editdistance-0.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc5f0c7f12a3a3bf2d129e2900deaaa5e47203ef61918343ddc4b6c03e50f089"}, + {file = "editdistance-0.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98572c662fd7d425ff24acb8197ad4be7849558a48aebbc60012090bfda4dce9"}, + {file = "editdistance-0.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b35c647a8a17b77441e7b6111b74ae1016851589109e1efc990d27225b3217b"}, + {file = "editdistance-0.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2f56c0e006f6b5207985c1bdd62e1873e66bb06a60849cad32716cad1bb3ae40"}, + {file = "editdistance-0.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d6bc5a827b262dc9b0d03cfd821682334ce1280520edf6385dc1730e390b5201"}, + {file = "editdistance-0.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ad68a2357664e45823b38c9f67a315ff9771263ec502a710057b78c6ca6fcfcd"}, + {file = "editdistance-0.8.1-cp39-cp39-win32.whl", hash = "sha256:16b3e413c020e42b2ef2d4ba01386ead43007217f0bdd704e90474ace90d2023"}, + {file = "editdistance-0.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:331f1a8d3a753858a9d689c0bcd79ad1959e0df464bb6c22cb263cfb6da208e4"}, + {file = "editdistance-0.8.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a4a90c6b03094c07358572027a8d0a13cca7450b1aa6caca98a5f1fa4f0b8961"}, + {file = "editdistance-0.8.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:510a4f9ced348a4fd89ae2e102357d4d801a771e29bb2bc2f130a1692193407f"}, + {file = "editdistance-0.8.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4787fa7228ba6a34b430066d174320f011d605015baa7299c2c4911e6ea6bd46"}, + {file = "editdistance-0.8.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee02601375073afccd6b4d811129ce1cb696d47db734784d8dbd1fddcea75447"}, + {file = "editdistance-0.8.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:bc7ad9f9a20e6f351523de77c59249f005242e3f317b5de45d02c378d24f6531"}, + {file = "editdistance-0.8.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7743895df48482fa5a7136543d6bde72e6c10c78a4a4b772fcddda48f792ef68"}, + {file = "editdistance-0.8.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9f139d921aff07deb2c9e592fe23d994af0e59267962a20c062cd66750a0ca4"}, + {file = "editdistance-0.8.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:493587829de3e500bdf34f03f5ab12501867b911acc838e1d04047a3f8941aad"}, + {file = "editdistance-0.8.1-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16c521157f5d29bf2cb20472482de8450722685d27a6dd801ff1e80cb13a1fd1"}, + {file = "editdistance-0.8.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:2a956eb9584d9e8d165bddd9091791924648071c3cdb1e03ec94b1320c2edefd"}, + {file = "editdistance-0.8.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8131acb6b5170382b8b74efab92df8739ac591dc841314e0153af63c4493cb43"}, + {file = "editdistance-0.8.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f182e5e1d2a446138cab085409395c62af36eb1abcbe8cfacb083febfeafd5ce"}, + {file = "editdistance-0.8.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f4f19a829aff230377041acb77afec73becbebafe35b7e322be00cdb3122ddb"}, + {file = "editdistance-0.8.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b978c5927100a57791131dd2418040f4e5d33970d37b97a84c1a530ec481f557"}, + {file = "editdistance-0.8.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:0c96a8e981f385f0b7392d047c5caab8e0b24f94b71120787fd78241efc34237"}, + {file = "editdistance-0.8.1.tar.gz", hash = "sha256:d1cdf80a5d5014b0c9126a69a42ce55a457b457f6986ff69ca98e4fe4d2d8fed"}, +] + [[package]] name = "einops" version = "0.8.2" @@ -1391,7 +1564,7 @@ description = "A robust email address syntax and deliverability validation libra optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4"}, {file = "email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426"}, @@ -1401,6 +1574,115 @@ files = [ dnspython = ">=2.0.0" idna = ">=2.0.0" +[[package]] +name = "evalscope" +version = "1.11.1" +description = "EvalScope: Lightweight LLMs Evaluation Framework" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "evalscope-1.11.1-py3-none-any.whl", hash = "sha256:5058c5112ee0dfff0048a48ee1ff9c6e1ca28bbed39ef5cd34a6a14d3ddf2b66"}, + {file = "evalscope-1.11.1.tar.gz", hash = "sha256:4a6f4437a37543a608a08f5279eb215a2832d32d1ecc812ba8442a67c3340460"}, +] + +[package.dependencies] +aiohttp = "*" +colorlog = "*" +docstring_parser = "*" +dotenv = "*" +editdistance = "*" +filetype = "*" +jieba = "*" +jinja2 = "*" +jsonlines = "*" +jsonschema = "*" +latex2sympy2_extended = {version = "*", extras = ["antlr4-9-3"]} +litellm = ">=1.55,<2.0" +Markdown = "*" +modelscope = {version = ">=1.34", extras = ["datasets"]} +more_itertools = "*" +nltk = "*" +openai = "*" +overrides = "*" +pandas = "*" +pillow = "*" +plotly = "*" +pydantic = "*" +pylatexenc = "*" +pyyaml = ">=5.1" +requests = "*" +rich = "*" +rouge-chinese = "*" +rouge-score = ">=0.1.0" +sacrebleu = "*" +sympy = "*" +tabulate = "*" +tqdm = "*" +transformers = ">=4.33,<4.57.2 || >4.57.2" +word2number = "*" +zhconv = "*" + +[package.extras] +aigc = ["diffusers", "iopath", "lpips", "omegaconf", "open_clip_torch", "opencv-python", "peft (>=0.17)", "torch", "torchvision"] +air-bench = ["soundfile"] +all = ["aiohttp", "diffusers", "fastapi", "flask (>=2.0.0)", "flask (>=2.0.0)", "instructor", "iopath", "jinja2", "langchain (>=0.3.0)", "langchain-community (>=0.3.0)", "langchain-core (>=0.3.0)", "langchain-huggingface (>=0.1.0)", "langchain-openai (>=0.3.0)", "lpips", "modelscope[datasets] (>=1.10.0)", "ms-opencompass (>=0.1.6)", "ms-vlmeval (>=0.0.17)", "mteb (>=2.7.0,<3.0.0)", "numpy", "omegaconf", "open_clip_torch", "openai (>=1.0.0)", "opencv-python", "peft (>=0.17)", "plotly (>=5.23.0)", "plotly (>=5.23.0)", "ragas (>=0.4.0,<0.5.0)", "rich", "sentence-transformers (>=5.4.0)", "sse_starlette", "torch", "torch (>=2.0.0)", "torchvision", "transformers", "unstructured", "uvicorn", "uvloop ; sys_platform != \"win32\"", "webdataset (>0.2.0)"] +app = ["flask (>=2.0.0)", "plotly (>=5.23.0)"] +arena-hard = ["scikit-learn"] +bfcl = ["bfcl-eval (==2025.10.27.1)"] +caption = ["pycocoevalcap"] +cc-ocr-v2 = ["apted", "distance", "lxml", "python-Levenshtein", "scipy", "zss"] +deep-swe = ["datacurve-pier (>=0.3.0)"] +dev = ["hypothesis (==6.155.2)", "pre-commit (==4.6.0)", "pytest", "pytest-cov", "python-dotenv"] +docs = ["docutils (>=0.16.0)", "myst_parser", "recommonmark", "sphinx (>=5.3.0)", "sphinx-design", "sphinxawesome-theme"] +general-arena = ["scikit-learn"] +ifbench = ["emoji", "nltk (>=3.9)", "syllapy"] +ifeval = ["langdetect", "nltk (>=3.9)"] +maritime-ocr-bench = ["numpy", "pillow", "shapely"] +mcp = ["mcp (>=1.0)", "mcp-server-fetch"] +miniwob = ["browsergym-miniwob (==0.14.3)"] +multi-if = ["emoji", "langdetect", "nltk (>=3.9)", "pythainlp"] +needle-haystack = ["matplotlib", "seaborn"] +ocr-bench = ["Polygon3", "apted", "distance", "editdistance", "jieba", "lxml", "nltk (>=3.9)", "python-Levenshtein", "zss"] +olmocr-bench = ["beautifulsoup4", "fuzzysearch", "rapidfuzz"] +olympiad-bench = ["latex2sympy2_extended[antlr4-9-3]"] +omnidoc-bench = ["apted", "beautifulsoup4", "distance", "jieba", "lxml", "python-Levenshtein"] +openai-mrcr = ["tiktoken"] +opencompass = ["ms-opencompass (>=0.1.6)"] +perf = ["aiohttp", "fastapi", "jinja2", "numpy", "rich", "sse_starlette", "transformers", "uvicorn", "uvloop ; sys_platform != \"win32\""] +rag = ["instructor", "langchain (>=0.3.0)", "langchain-community (>=0.3.0)", "langchain-core (>=0.3.0)", "langchain-huggingface (>=0.1.0)", "langchain-openai (>=0.3.0)", "modelscope[datasets] (>=1.10.0)", "mteb (>=2.7.0,<3.0.0)", "openai (>=1.0.0)", "ragas (>=0.4.0,<0.5.0)", "sentence-transformers (>=5.4.0)", "torch (>=2.0.0)", "unstructured", "webdataset (>0.2.0)"] +refcoco = ["pycocoevalcap"] +sandbox = ["ms-enclave[docker] (>=0.0.8)"] +service = ["aiohttp", "fastapi", "flask (>=2.0.0)", "jinja2", "numpy", "plotly (>=5.23.0)", "rich", "sse_starlette", "transformers", "uvicorn", "uvloop ; sys_platform != \"win32\""] +swe-bench = ["swebench (==4.1.0)"] +terminal-bench = ["harbor (>=0.8.0,<1.0.0)"] +toolathlon = ["httpx", "websockets"] +torgo = ["jellyfish", "jiwer"] +vlmeval = ["ms-vlmeval (>=0.0.17)"] +vtcbench = ["beautifulsoup4"] +wide-search = ["dateparser"] +wmt = ["unbabel-comet"] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version == \"3.10\"" +files = [ + {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, + {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "fastapi" version = "0.135.1" @@ -1440,7 +1722,7 @@ description = "Run and manage FastAPI apps from the command line with FastAPI CL optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "fastapi_cli-0.0.24-py3-none-any.whl", hash = "sha256:4a1f78ed798f106b4fee85ca93b85d8fe33c0a3570f775964d37edb80b8f0edc"}, {file = "fastapi_cli-0.0.24.tar.gz", hash = "sha256:1afc9c9e21d7ebc8a3ca5e31790cd8d837742be7e4f8b9236e99cb3451f0de00"}, @@ -1449,6 +1731,7 @@ files = [ [package.dependencies] fastapi-cloud-cli = {version = ">=0.1.1", optional = true, markers = "extra == \"standard\""} rich-toolkit = ">=0.14.8" +tomli = {version = ">=2.0.0", markers = "python_version < \"3.11\""} typer = ">=0.16.0" uvicorn = {version = ">=0.15.0", extras = ["standard"]} @@ -1464,7 +1747,7 @@ description = "Deploy and manage FastAPI Cloud apps from the command line 🚀" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "fastapi_cloud_cli-0.14.0-py3-none-any.whl", hash = "sha256:325fcb4b45e661184152da6db861d9fb718739fbcd561a4d334dbe78c026586f"}, {file = "fastapi_cloud_cli-0.14.0.tar.gz", hash = "sha256:d3ecb8c942685a71df0af7bd59f463b5eff76f5818b48e5a03c6159726831e68"}, @@ -1473,7 +1756,10 @@ files = [ [package.dependencies] fastar = ">=0.8.0" httpx = ">=0.27.0" -pydantic = {version = ">=2.7.4", extras = ["email"], markers = "python_version < \"3.13\""} +pydantic = [ + {version = ">=2.7.4", extras = ["email"], markers = "python_version < \"3.13\""}, + {version = ">=2.8.0", extras = ["email"], markers = "python_version == \"3.13\""}, +] rich-toolkit = ">=0.19.7" rignore = ">=0.5.1" sentry-sdk = ">=2.20.0" @@ -1490,7 +1776,7 @@ description = "High-level bindings for the Rust tar crate" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "fastar-0.8.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c9f930cff014cf79d396d0541bd9f3a3f170c9b5e45d10d634d98f9ed08788c3"}, {file = "fastar-0.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:07b70f712d20622346531a4b46bb332569bea621f61314c0b7e80903a16d14cf"}, @@ -1648,6 +1934,95 @@ files = [ {file = "fastar-0.8.0.tar.gz", hash = "sha256:f4d4d68dbf1c4c2808f0e730fac5843493fc849f70fe3ad3af60dfbaf68b9a12"}, ] +[[package]] +name = "fastuuid" +version = "0.14.0" +description = "Python bindings to Rust's UUID library." +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "fastuuid-0.14.0-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:6e6243d40f6c793c3e2ee14c13769e341b90be5ef0c23c82fa6515a96145181a"}, + {file = "fastuuid-0.14.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:13ec4f2c3b04271f62be2e1ce7e95ad2dd1cf97e94503a3760db739afbd48f00"}, + {file = "fastuuid-0.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b2fdd48b5e4236df145a149d7125badb28e0a383372add3fbaac9a6b7a394470"}, + {file = "fastuuid-0.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f74631b8322d2780ebcf2d2d75d58045c3e9378625ec51865fe0b5620800c39d"}, + {file = "fastuuid-0.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83cffc144dc93eb604b87b179837f2ce2af44871a7b323f2bfed40e8acb40ba8"}, + {file = "fastuuid-0.14.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a771f135ab4523eb786e95493803942a5d1fc1610915f131b363f55af53b219"}, + {file = "fastuuid-0.14.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4edc56b877d960b4eda2c4232f953a61490c3134da94f3c28af129fb9c62a4f6"}, + {file = "fastuuid-0.14.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bcc96ee819c282e7c09b2eed2b9bd13084e3b749fdb2faf58c318d498df2efbe"}, + {file = "fastuuid-0.14.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7a3c0bca61eacc1843ea97b288d6789fbad7400d16db24e36a66c28c268cfe3d"}, + {file = "fastuuid-0.14.0-cp310-cp310-win32.whl", hash = "sha256:7f2f3efade4937fae4e77efae1af571902263de7b78a0aee1a1653795a093b2a"}, + {file = "fastuuid-0.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:ae64ba730d179f439b0736208b4c279b8bc9c089b102aec23f86512ea458c8a4"}, + {file = "fastuuid-0.14.0-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:73946cb950c8caf65127d4e9a325e2b6be0442a224fd51ba3b6ac44e1912ce34"}, + {file = "fastuuid-0.14.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:12ac85024637586a5b69645e7ed986f7535106ed3013640a393a03e461740cb7"}, + {file = "fastuuid-0.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:05a8dde1f395e0c9b4be515b7a521403d1e8349443e7641761af07c7ad1624b1"}, + {file = "fastuuid-0.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09378a05020e3e4883dfdab438926f31fea15fd17604908f3d39cbeb22a0b4dc"}, + {file = "fastuuid-0.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbb0c4b15d66b435d2538f3827f05e44e2baafcc003dd7d8472dc67807ab8fd8"}, + {file = "fastuuid-0.14.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cd5a7f648d4365b41dbf0e38fe8da4884e57bed4e77c83598e076ac0c93995e7"}, + {file = "fastuuid-0.14.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c0a94245afae4d7af8c43b3159d5e3934c53f47140be0be624b96acd672ceb73"}, + {file = "fastuuid-0.14.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2b29e23c97e77c3a9514d70ce343571e469098ac7f5a269320a0f0b3e193ab36"}, + {file = "fastuuid-0.14.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1e690d48f923c253f28151b3a6b4e335f2b06bf669c68a02665bc150b7839e94"}, + {file = "fastuuid-0.14.0-cp311-cp311-win32.whl", hash = "sha256:a6f46790d59ab38c6aa0e35c681c0484b50dc0acf9e2679c005d61e019313c24"}, + {file = "fastuuid-0.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:e150eab56c95dc9e3fefc234a0eedb342fac433dacc273cd4d150a5b0871e1fa"}, + {file = "fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77e94728324b63660ebf8adb27055e92d2e4611645bf12ed9d88d30486471d0a"}, + {file = "fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:caa1f14d2102cb8d353096bc6ef6c13b2c81f347e6ab9d6fbd48b9dea41c153d"}, + {file = "fastuuid-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d23ef06f9e67163be38cece704170486715b177f6baae338110983f99a72c070"}, + {file = "fastuuid-0.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c9ec605ace243b6dbe3bd27ebdd5d33b00d8d1d3f580b39fdd15cd96fd71796"}, + {file = "fastuuid-0.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:808527f2407f58a76c916d6aa15d58692a4a019fdf8d4c32ac7ff303b7d7af09"}, + {file = "fastuuid-0.14.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fb3c0d7fef6674bbeacdd6dbd386924a7b60b26de849266d1ff6602937675c8"}, + {file = "fastuuid-0.14.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab3f5d36e4393e628a4df337c2c039069344db5f4b9d2a3c9cea48284f1dd741"}, + {file = "fastuuid-0.14.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b9a0ca4f03b7e0b01425281ffd44e99d360e15c895f1907ca105854ed85e2057"}, + {file = "fastuuid-0.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3acdf655684cc09e60fb7e4cf524e8f42ea760031945aa8086c7eae2eeeabeb8"}, + {file = "fastuuid-0.14.0-cp312-cp312-win32.whl", hash = "sha256:9579618be6280700ae36ac42c3efd157049fe4dd40ca49b021280481c78c3176"}, + {file = "fastuuid-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:d9e4332dc4ba054434a9594cbfaf7823b57993d7d8e7267831c3e059857cf397"}, + {file = "fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77a09cb7427e7af74c594e409f7731a0cf887221de2f698e1ca0ebf0f3139021"}, + {file = "fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9bd57289daf7b153bfa3e8013446aa144ce5e8c825e9e366d455155ede5ea2dc"}, + {file = "fastuuid-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ac60fc860cdf3c3f327374db87ab8e064c86566ca8c49d2e30df15eda1b0c2d5"}, + {file = "fastuuid-0.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab32f74bd56565b186f036e33129da77db8be09178cd2f5206a5d4035fb2a23f"}, + {file = "fastuuid-0.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33e678459cf4addaedd9936bbb038e35b3f6b2061330fd8f2f6a1d80414c0f87"}, + {file = "fastuuid-0.14.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e3cc56742f76cd25ecb98e4b82a25f978ccffba02e4bdce8aba857b6d85d87b"}, + {file = "fastuuid-0.14.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cb9a030f609194b679e1660f7e32733b7a0f332d519c5d5a6a0a580991290022"}, + {file = "fastuuid-0.14.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:09098762aad4f8da3a888eb9ae01c84430c907a297b97166b8abc07b640f2995"}, + {file = "fastuuid-0.14.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1383fff584fa249b16329a059c68ad45d030d5a4b70fb7c73a08d98fd53bcdab"}, + {file = "fastuuid-0.14.0-cp313-cp313-win32.whl", hash = "sha256:a0809f8cc5731c066c909047f9a314d5f536c871a7a22e815cc4967c110ac9ad"}, + {file = "fastuuid-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:0df14e92e7ad3276327631c9e7cec09e32572ce82089c55cb1bb8df71cf394ed"}, + {file = "fastuuid-0.14.0-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:b852a870a61cfc26c884af205d502881a2e59cc07076b60ab4a951cc0c94d1ad"}, + {file = "fastuuid-0.14.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c7502d6f54cd08024c3ea9b3514e2d6f190feb2f46e6dbcd3747882264bb5f7b"}, + {file = "fastuuid-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1ca61b592120cf314cfd66e662a5b54a578c5a15b26305e1b8b618a6f22df714"}, + {file = "fastuuid-0.14.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa75b6657ec129d0abded3bec745e6f7ab642e6dba3a5272a68247e85f5f316f"}, + {file = "fastuuid-0.14.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8a0dfea3972200f72d4c7df02c8ac70bad1bb4c58d7e0ec1e6f341679073a7f"}, + {file = "fastuuid-0.14.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1bf539a7a95f35b419f9ad105d5a8a35036df35fdafae48fb2fd2e5f318f0d75"}, + {file = "fastuuid-0.14.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:9a133bf9cc78fdbd1179cb58a59ad0100aa32d8675508150f3658814aeefeaa4"}, + {file = "fastuuid-0.14.0-cp314-cp314-musllinux_1_1_i686.whl", hash = "sha256:f54d5b36c56a2d5e1a31e73b950b28a0d83eb0c37b91d10408875a5a29494bad"}, + {file = "fastuuid-0.14.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:ec27778c6ca3393ef662e2762dba8af13f4ec1aaa32d08d77f71f2a70ae9feb8"}, + {file = "fastuuid-0.14.0-cp314-cp314-win32.whl", hash = "sha256:e23fc6a83f112de4be0cc1990e5b127c27663ae43f866353166f87df58e73d06"}, + {file = "fastuuid-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:df61342889d0f5e7a32f7284e55ef95103f2110fee433c2ae7c2c0956d76ac8a"}, + {file = "fastuuid-0.14.0-cp38-cp38-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:47c821f2dfe95909ead0085d4cb18d5149bca704a2b03e03fb3f81a5202d8cea"}, + {file = "fastuuid-0.14.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3964bab460c528692c70ab6b2e469dd7a7b152fbe8c18616c58d34c93a6cf8d4"}, + {file = "fastuuid-0.14.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c501561e025b7aea3508719c5801c360c711d5218fc4ad5d77bf1c37c1a75779"}, + {file = "fastuuid-0.14.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dce5d0756f046fa792a40763f36accd7e466525c5710d2195a038f93ff96346"}, + {file = "fastuuid-0.14.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193ca10ff553cf3cc461572da83b5780fc0e3eea28659c16f89ae5202f3958d4"}, + {file = "fastuuid-0.14.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0737606764b29785566f968bd8005eace73d3666bd0862f33a760796e26d1ede"}, + {file = "fastuuid-0.14.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e0976c0dff7e222513d206e06341503f07423aceb1db0b83ff6851c008ceee06"}, + {file = "fastuuid-0.14.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6fbc49a86173e7f074b1a9ec8cf12ca0d54d8070a85a06ebf0e76c309b84f0d0"}, + {file = "fastuuid-0.14.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:de01280eabcd82f7542828ecd67ebf1551d37203ecdfd7ab1f2e534edb78d505"}, + {file = "fastuuid-0.14.0-cp38-cp38-win32.whl", hash = "sha256:af5967c666b7d6a377098849b07f83462c4fedbafcf8eb8bc8ff05dcbe8aa209"}, + {file = "fastuuid-0.14.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3091e63acf42f56a6f74dc65cfdb6f99bfc79b5913c8a9ac498eb7ca09770a8"}, + {file = "fastuuid-0.14.0-cp39-cp39-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2ec3d94e13712a133137b2805073b65ecef4a47217d5bac15d8ac62376cefdb4"}, + {file = "fastuuid-0.14.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:139d7ff12bb400b4a0c76be64c28cbe2e2edf60b09826cbfd85f33ed3d0bbe8b"}, + {file = "fastuuid-0.14.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d55b7e96531216fc4f071909e33e35e5bfa47962ae67d9e84b00a04d6e8b7173"}, + {file = "fastuuid-0.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0eb25f0fd935e376ac4334927a59e7c823b36062080e2e13acbaf2af15db836"}, + {file = "fastuuid-0.14.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:089c18018fdbdda88a6dafd7d139f8703a1e7c799618e33ea25eb52503d28a11"}, + {file = "fastuuid-0.14.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fc37479517d4d70c08696960fad85494a8a7a0af4e93e9a00af04d74c59f9e3"}, + {file = "fastuuid-0.14.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:73657c9f778aba530bc96a943d30e1a7c80edb8278df77894fe9457540df4f85"}, + {file = "fastuuid-0.14.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d31f8c257046b5617fc6af9c69be066d2412bdef1edaa4bdf6a214cf57806105"}, + {file = "fastuuid-0.14.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5816d41f81782b209843e52fdef757a361b448d782452d96abedc53d545da722"}, + {file = "fastuuid-0.14.0-cp39-cp39-win32.whl", hash = "sha256:448aa6833f7a84bfe37dd47e33df83250f404d591eb83527fa2cac8d1e57d7f3"}, + {file = "fastuuid-0.14.0-cp39-cp39-win_amd64.whl", hash = "sha256:84b0779c5abbdec2a9511d5ffbfcd2e53079bf889824b32be170c0d8ef5fc74c"}, + {file = "fastuuid-0.14.0.tar.gz", hash = "sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26"}, +] + [[package]] name = "filelock" version = "3.25.0" @@ -1660,6 +2035,19 @@ files = [ {file = "filelock-3.25.0.tar.gz", hash = "sha256:8f00faf3abf9dc730a1ffe9c354ae5c04e079ab7d3a683b7c32da5dd05f26af3"}, ] +[[package]] +name = "filetype" +version = "1.2.0" +description = "Infer file type and MIME type of any file/buffer. No external dependencies." +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25"}, + {file = "filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb"}, +] + [[package]] name = "flashinfer-python" version = "0.6.3" @@ -1667,7 +2055,7 @@ description = "FlashInfer: Kernel Library for LLM Serving" optional = true python-versions = "<4.0,>=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "flashinfer_python-0.6.3-py3-none-any.whl", hash = "sha256:0fe2de934a4b3690c543dafb03f38d7bb4a762431abe8ae4f7292d6fef10c65d"}, {file = "flashinfer_python-0.6.3.tar.gz", hash = "sha256:84a762538247a86bc52ff31d9505d161ce1ec059174c1821c87c3ed1e44670fc"}, @@ -1878,7 +2266,7 @@ description = "Read and write ML models in GGUF for GGML" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "gguf-0.18.0-py3-none-any.whl", hash = "sha256:af93f7ef198a265cbde5fa6a6b3101528bca285903949ab0a3e591cd993a1864"}, {file = "gguf-0.18.0.tar.gz", hash = "sha256:b4659093d5d0dccdb5902a904d54b327f4052879fe5e90946ad5fce9f8018c2e"}, @@ -1900,7 +2288,7 @@ description = "Google API client core library" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "google_api_core-2.30.0-py3-none-any.whl", hash = "sha256:80be49ee937ff9aba0fd79a6eddfde35fe658b9953ab9b79c57dd7061afa8df5"}, {file = "google_api_core-2.30.0.tar.gz", hash = "sha256:02edfa9fab31e17fc0befb5f161b3bf93c9096d99aed584625f38065c511ad9b"}, @@ -1909,7 +2297,10 @@ files = [ [package.dependencies] google-auth = ">=2.14.1,<3.0.0" googleapis-common-protos = ">=1.56.3,<2.0.0" -proto-plus = ">=1.22.3,<2.0.0" +proto-plus = [ + {version = ">=1.22.3,<2.0.0"}, + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, +] protobuf = ">=4.25.8,<7.0.0" requests = ">=2.20.0,<3.0.0" @@ -1924,7 +2315,7 @@ description = "Google Authentication Library" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "google_auth-2.48.0-py3-none-any.whl", hash = "sha256:2e2a537873d449434252a9632c28bfc268b0adb1e53f9fb62afc5333a975903f"}, {file = "google_auth-2.48.0.tar.gz", hash = "sha256:4f7e706b0cd3208a3d940a19a822c37a476ddba5450156c3e6624a71f7c841ce"}, @@ -1953,7 +2344,7 @@ description = "Common protobufs used in Google APIs" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" files = [ {file = "googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038"}, {file = "googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5"}, @@ -1972,7 +2363,7 @@ description = "HTTP/2-based RPC framework" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"server\" or extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "grpcio-1.78.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:7cc47943d524ee0096f973e1081cb8f4f17a4615f2116882a5f1416e4cfe92b5"}, {file = "grpcio-1.78.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:c3f293fdc675ccba4db5a561048cca627b5e7bd1c8a6973ffedabe7d116e22e2"}, @@ -2050,7 +2441,7 @@ description = "Standard Protobuf Reflection Service for gRPC" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "grpcio_reflection-1.78.0-py3-none-any.whl", hash = "sha256:06fcfde9e6888cdd12e9dd1cf6dc7c440c2e9acf420f696ccbe008672ed05b60"}, {file = "grpcio_reflection-1.78.0.tar.gz", hash = "sha256:e6e60c0b85dbcdf963b4d4d150c0f1d238ba891d805b575c52c0365d07fc0c40"}, @@ -2067,7 +2458,7 @@ description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\" or extra == \"tinker\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" files = [ {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, @@ -2080,7 +2471,7 @@ description = "Pure-Python HTTP/2 protocol implementation" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"tinker\"" +markers = "python_version >= \"3.11\" and extra == \"client\"" files = [ {file = "h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd"}, {file = "h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1"}, @@ -2136,7 +2527,7 @@ description = "Pure-Python HPACK header encoding" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"tinker\"" +markers = "python_version >= \"3.11\" and extra == \"client\"" files = [ {file = "hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496"}, {file = "hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca"}, @@ -2149,7 +2540,7 @@ description = "A minimal low-level HTTP client." optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"tinker\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" files = [ {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, @@ -2172,7 +2563,7 @@ description = "A collection of framework independent HTTP protocol utils." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "httptools-0.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:11d01b0ff1fe02c4c32d60af61a4d613b74fad069e47e06e9067758c01e9ac78"}, {file = "httptools-0.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:84d86c1e5afdc479a6fdabf570be0d3eb791df0ae727e8dbc0259ed1249998d4"}, @@ -2226,7 +2617,7 @@ description = "The next generation HTTP client." optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"tinker\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" files = [ {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, @@ -2253,7 +2644,7 @@ description = "Consume Server-Sent Event (SSE) messages with HTTPX." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc"}, {file = "httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d"}, @@ -2298,6 +2689,24 @@ testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1. torch = ["safetensors[torch]", "torch"] typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] +[[package]] +name = "hydra-core" +version = "1.3.7" +description = "A framework for elegantly configuring complex applications" +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "hydra_core-1.3.7-py3-none-any.whl", hash = "sha256:83606c4a05c770f1e026e0489875f28989829d151799eabd8cfa8cbe82164515"}, + {file = "hydra_core-1.3.7.tar.gz", hash = "sha256:d4ff629c7e23b64a9db5fe5b027cf7c55b633b737f03be51ef41b77ed016f75e"}, +] + +[package.dependencies] +antlr4-python3-runtime = "==4.9.*" +omegaconf = ">=2.2,<2.4" +packaging = "*" + [[package]] name = "hyperframe" version = "6.1.0" @@ -2305,12 +2714,126 @@ description = "Pure-Python HTTP/2 framing" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"tinker\"" +markers = "python_version >= \"3.11\" and extra == \"client\"" files = [ {file = "hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5"}, {file = "hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08"}, ] +[[package]] +name = "hypothesis" +version = "6.168.0" +description = "The property-based testing library for Python" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"test\"" +files = [ + {file = "hypothesis-6.168.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:47b89491ff02e3ae9b302c440457938e87b47a45b9a1d98ff5575b6910d779e2"}, + {file = "hypothesis-6.168.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:1f4cd0ff11bd470a1a846296ed5fe55e84214194850370994fd1370fe73d3099"}, + {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:732ae5d47482f99d8028cca096729625f05690a83f5e7ce31466e266155792f4"}, + {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2085ee74ac3ab6b70e2f7ffae9b4cb74c246da2f574b2de81a0818a8a30f659f"}, + {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1894782fae5d9a7bb44e6dcf848ccb09ccb5babab48d8b5c31a0a7fc025b82a1"}, + {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecf0ab13cef899efb816ffdd7963e0679f372520884ce06756c7642f3df94213"}, + {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:3f6dcf66270278d078bed01b401f47db4e26456cd909d8e23c6b9366a6c0b131"}, + {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bfef4d46dbf1704a7b8fa3a78778651a2cb18870ca0a70da19c381646822b149"}, + {file = "hypothesis-6.168.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1d1aa5b3484e329295d88488a5ba06243909e65c2ab616513c2d36721de4ed1d"}, + {file = "hypothesis-6.168.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3bc00fd8cda04b58e37a1163e8a65389b247b4f5ee547ae37d244a4960995517"}, + {file = "hypothesis-6.168.0-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:990026952d5b2eca290c88f639ac639233f47e13dae338c6dfb6e4774bcab349"}, + {file = "hypothesis-6.168.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:a74b0945acbbd552c7c2d0a99a3b5232962b8848c8eed1829451800a9bfcf00b"}, + {file = "hypothesis-6.168.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2a380b521b5a76a9e8917d64adcf7f861a45a4360a34b1579af14c5df8eb0377"}, + {file = "hypothesis-6.168.0-cp310-abi3-win32.whl", hash = "sha256:2264f15a1c80329e3ad48e39c44bd5c9429b7b04c9ee62cdd72f4b10aaac9f29"}, + {file = "hypothesis-6.168.0-cp310-abi3-win_amd64.whl", hash = "sha256:5b54769033b84477931d2072e7133a7555e0de5c53fd5ca3bbde960762d7d31b"}, + {file = "hypothesis-6.168.0-cp310-abi3-win_arm64.whl", hash = "sha256:112b0900059bf9d7d6528ed729770629ab146e0d133c4143b9bd4a01dc002bcc"}, + {file = "hypothesis-6.168.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:cb10aa59b0af45badca76911f5323f40d24fdbe00d01b7b67fef8648c99411b5"}, + {file = "hypothesis-6.168.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e21e30b76b6d3adb87c550576132a3204f4c257ec43353f6c09b9d59bb762abc"}, + {file = "hypothesis-6.168.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:527452b43e79e6dfbf9cb69145a940547a3cd177c556698a3fc939ed2354c4b3"}, + {file = "hypothesis-6.168.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:489d5c060f49f495b64215cae627c71730cffd5ef59dc4d7f431932e6e6d2e67"}, + {file = "hypothesis-6.168.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c3af200b322f710c76c2189866246cdcff2039165dd77edff1a7bf1157162fb0"}, + {file = "hypothesis-6.168.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:73084b76e4a79cd0f7883ce80fc60c9f374ce7dcad8f520b39db40470ce1852f"}, + {file = "hypothesis-6.168.0-cp310-cp310-win_amd64.whl", hash = "sha256:8067e6b4b48e5cfdc849a1a20c9d4972b3f532b3e3edb5e2b5dfd106045a5236"}, + {file = "hypothesis-6.168.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4d7d29dd63ad9fdc4aa1d65fa272449e14aaf6c6bb8451091818c2945533a43a"}, + {file = "hypothesis-6.168.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9a72ed7afa1f7e30488b8a5754fca0ad9755518bdb77d6f0b003cadf7437a5f9"}, + {file = "hypothesis-6.168.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcc5bad4300a751804ce41f0e10d77f85272668160708ce39ec579bca8984843"}, + {file = "hypothesis-6.168.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53469a1a7c4861b12c9a8622f762d7d1fd7bcf171884e1018ed5a8f063a5c063"}, + {file = "hypothesis-6.168.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a9650c4882fdbdd8e90bdae602a8bfa8c6f09dc5d06afec5b9b23982e8f60a04"}, + {file = "hypothesis-6.168.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:348d9b93fd4129f67f9bab94f3d70709a9372bbe0e0d22731325ce85d5eb409f"}, + {file = "hypothesis-6.168.0-cp311-cp311-win_amd64.whl", hash = "sha256:719b45b0512e3535a6a0077c2f7c6053b02ac0e72d60693f66f98790a33855b2"}, + {file = "hypothesis-6.168.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6b750390dac4429da0cb70ab3fe758457f0cea3d9c843d48c59d0690d1189fda"}, + {file = "hypothesis-6.168.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8e4b2d434e0dd134f3d31ac1efc1825bf99730dfe70fec005ff66d7211836d79"}, + {file = "hypothesis-6.168.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d4d36ed2fd62de11382f1d608169c1ffa9a49d3b9351146d8ff87cb81a66f7"}, + {file = "hypothesis-6.168.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5920d267f7d8cfd376672f2bde5905cdf284d47519582e41ce7c142d48ee46c4"}, + {file = "hypothesis-6.168.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fb8cdf45361e259df86e19f8cd042ce2d6c7e6ad88fa631b78a4e3a83c2e572d"}, + {file = "hypothesis-6.168.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b3ce1cce70b25a37ed1a38a53ce7204785726c675c0f41a0f83c338a7e47b3d"}, + {file = "hypothesis-6.168.0-cp312-cp312-win_amd64.whl", hash = "sha256:f62bdabf278db9ff61df5f3203d608949f0d893d0e30cdac3f2330e67e41ae68"}, + {file = "hypothesis-6.168.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d55562bf8d41cfa18559c33f30cadf44ceac8e517509d7a022a9feace621f28"}, + {file = "hypothesis-6.168.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92cff497b92e2285ff6a94193fdee04aba483a4115d501c1f9a570bd103fcd20"}, + {file = "hypothesis-6.168.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ff259260015f9be3756dcd4bc11c08e007314dec6b43d9a89084c4f34f94475"}, + {file = "hypothesis-6.168.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35f1262831b5acc74ded15f629965daffcd657f6016ee04fc9605f6eb2b334c0"}, + {file = "hypothesis-6.168.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:046fe4bcfce2a2fa186ba9d96bbb62c25c2f6c2e4071f0783ed6b5cc481d0669"}, + {file = "hypothesis-6.168.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:24b52a2b1c8db6e1e516f9295c8e4ef7ef63303ff24fbbc5b35f4ff71dcd732c"}, + {file = "hypothesis-6.168.0-cp313-cp313-win_amd64.whl", hash = "sha256:ec0886fe0be9091669937989f9a662beca42ae14a4a6dab25491c2c63365f88d"}, + {file = "hypothesis-6.168.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e2df8afacf9261070795db36db4a394e3ccdbb663fd2d38c7a9fba0c836dcecc"}, + {file = "hypothesis-6.168.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9ba679f183c67adcb6f4ad93694beafb6da99fe691757f4e57b04ae77e581ba8"}, + {file = "hypothesis-6.168.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d9a8574f80fc859313aee56167d202e8625c0eedd200971130f0839f06d1c93"}, + {file = "hypothesis-6.168.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:deb02de608268928d779aa889b0a9d67794b1cc0c54a322cf19e386be8a46ca7"}, + {file = "hypothesis-6.168.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:076a2096c34448931c3cfeb2eb7a6b843a56ffdce5e4e3a025bfdf8f935666d9"}, + {file = "hypothesis-6.168.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f099b1c8fc49ec2d9d7944e661addb97d7c38e818fb8d1f78073c43895a87f6"}, + {file = "hypothesis-6.168.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:93413d1b0af50a7b165d66278c529174bf2fd1773c78027735dc0b50d1d3fd27"}, + {file = "hypothesis-6.168.0-cp314-cp314-win_amd64.whl", hash = "sha256:db2751c27bffc8491a96d72969649089d5400115e4b7c49bf7167ebbdcc84193"}, + {file = "hypothesis-6.168.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:cd0c1dcf308e919c8ae708054d0ad61921ae87634a9aea574a9851da584cebc1"}, + {file = "hypothesis-6.168.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d0bdb77f976740b8cd5ec697327ea343d02d052b9916d213b5d4c65d823415cd"}, + {file = "hypothesis-6.168.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f7486bed33225d02f6aa78a4c4ba2b6f84992a82571cdda1bf08dce41d13507"}, + {file = "hypothesis-6.168.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ba3838c4a92e0b9730d1ed7e67e4950c152ad79d0a0c7594065262db84c55c4"}, + {file = "hypothesis-6.168.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:891b2d281ede45130e7fa0a22fd65336cc77ef2f780ec3792e8de6fc274a02c8"}, + {file = "hypothesis-6.168.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e86820053afad84677f301c0b892a226be1df49790800a65668ae7cc8a1ac571"}, + {file = "hypothesis-6.168.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a4956f41ab1ec6e6ef9262a35970e9f3e2caaaa1cdafe0d413156c6934dd99d8"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:754016594fe78cef91790e0922f60d183c52f531255fbfa30dac495b813e2128"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:6f0dd437ec01140676192422b61f2f833b3ce6a3213da9b7e196ad6b3777e795"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f77af7721ff35a58fa8797decd14c932c350a2548686c6e9b844db710a3a2441"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a0d28418c104d7268fdebcc09bc49f7b6569b5eb942430c6859f53ec8d4edf63"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:812a84c4cc7f7ae4fcb39a5647cc2698e6c18254f8423126425578f1dcdac782"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6de30e559eb151de14a5f74bceb4d97792a9315ada2a1816b5da825cd7d28edc"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:9018b20acdb061b2ef4b2fa7f558ca5db97ffea316e0a528bc003a24b2ac996e"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bc935a5d5f86fd8f5af951b8fbe00307f6f7c596f82a9a27c17d974f6ab0a26c"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:45fcfa05f746e253350f55f216bcef59754f5f2b85745f1fc2bb8ba81dd517a9"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:f89d8e998d3c936ffbbd1c3686c96f0378f6558aecc5967a3035a857f2bab0ad"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:d0620fa320fa66649e6bfd71e94f3f86115fffebb7e3c6dcece19d1aaff8e07f"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:4085b61e25d3dcc6c9151d4115269870aee8cdb921611ee5c989b2786449be09"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:b5449a64eb37d9a4aa6ac9cd2ab0fd1a24145adf421ef1536884f73f39824887"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-win32.whl", hash = "sha256:91e3de666a6c4f7543000d1710e25055d63ef3032c98bd2ab338b3087bdaa780"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:9a2079cd09919956dd388f1a1f8ea5a79f2b2437650fbeda31d8661217ffefef"}, + {file = "hypothesis-6.168.0-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:085c9aa246487c56a40ca89003d285cbffdbb5be4097ba6d0139f9c21003c04a"}, + {file = "hypothesis-6.168.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:16864797de4b024e4c6cebd44598af932f870aad811341bc5bc24c738801ff76"}, + {file = "hypothesis-6.168.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:283eda952bcb1987ccba1c8b634db0e8a960e1e92e2daa7003bc2392f19cea01"}, + {file = "hypothesis-6.168.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5427a3c951080c18170486f775df6a82153882b819eca6b8e7ed77693634e5ab"}, + {file = "hypothesis-6.168.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a838218ff1eab8d7b4bf66b96037fce0a802f61f2fa5fd4b784696cac365ce7"}, + {file = "hypothesis-6.168.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:34e3c8b66047ba92f8b8df5e427074058d92db58038f007da4bf9d14e934ad3c"}, + {file = "hypothesis-6.168.0.tar.gz", hash = "sha256:72af51087b7b5ab21c49f0d502f803c20897678652835596bd2a8b169a39135e"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.0", markers = "python_full_version < \"3.11.0\""} +sortedcontainers = ">=2.1.0,<3.0.0" + +[package.extras] +all = ["black (>=20.8b0)", "click (>=7.0)", "crosshair-tool (>=0.0.110)", "django (>=5.2)", "dpcontracts (>=0.4)", "hypothesis-crosshair (>=0.0.30)", "lark (>=0.10.1)", "libcst (>=0.3.16)", "numpy (>=1.21.6)", "pandas (>=1.1)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "tzdata (>=2026.3) ; sys_platform == \"emscripten\" or sys_platform == \"win32\"", "watchdog (>=4.0.0)"] +cli = ["black (>=20.8b0)", "click (>=7.0)", "rich (>=9.0.0)"] +codemods = ["libcst (>=0.3.16)"] +crosshair = ["crosshair-tool (>=0.0.110)", "hypothesis-crosshair (>=0.0.30)"] +dateutil = ["python-dateutil (>=1.4)"] +django = ["django (>=5.2)"] +dpcontracts = ["dpcontracts (>=0.4)"] +ghostwriter = ["black (>=20.8b0)"] +lark = ["lark (>=0.10.1)"] +numpy = ["numpy (>=1.21.6)"] +pandas = ["pandas (>=1.1)"] +pytest = ["pytest (>=4.6)"] +pytz = ["pytz (>=2014.1)"] +redis = ["redis (>=3.0.0)"] +watchdog = ["watchdog (>=4.0.0)"] +zoneinfo = ["tzdata (>=2026.3) ; sys_platform == \"emscripten\" or sys_platform == \"win32\""] + [[package]] name = "idna" version = "3.11" @@ -2333,7 +2856,7 @@ description = "Iterative JSON parser with standard Python iterator interfaces" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "ijson-3.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ea8dcac10d86adaeead454bc25c97b68d0bda573d5fd6f86f5e21cf8f7906f88"}, {file = "ijson-3.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:92b0495bbb2150bbf14fc5d98fb6d76bcd1c526605a172709e602e6fedc96495"}, @@ -2452,7 +2975,7 @@ description = "Read metadata from Python packages" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"megatron\" or extra == \"ray\"" +markers = "(python_version >= \"3.11\" or extra == \"eval\" or extra == \"megatron\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\") and (python_version == \"3.11\" or extra == \"eval\" or extra == \"megatron\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\" or extra == \"client\") and (extra == \"client\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\" or extra == \"eval\" or extra == \"megatron\")" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -2470,6 +2993,19 @@ perf = ["ipython"] test = ["flufl.flake8", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] type = ["mypy (<1.19) ; platform_python_implementation == \"PyPy\"", "pytest-mypy (>=1.0.1)"] +[[package]] +name = "iniconfig" +version = "2.3.0" +description = "brain-dead simple config-ini parsing" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"test\"" +files = [ + {file = "iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12"}, + {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, +] + [[package]] name = "interegular" version = "0.3.3" @@ -2477,12 +3013,24 @@ description = "a regex intersection checker" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "interegular-0.3.3-py37-none-any.whl", hash = "sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c"}, {file = "interegular-0.3.3.tar.gz", hash = "sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600"}, ] +[[package]] +name = "jieba" +version = "0.42.1" +description = "Chinese Words Segmentation Utilities" +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "jieba-0.42.1.tar.gz", hash = "sha256:055ca12f62674fafed09427f176506079bc135638a14e23e25be909131928db2"}, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -2508,7 +3056,7 @@ description = "Fast iterable JSON parser." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" files = [ {file = "jiter-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2ffc63785fd6c7977defe49b9824ae6ce2b2e2b77ce539bdaf006c26da06342e"}, {file = "jiter-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4a638816427006c1e3f0013eb66d391d7a3acda99a7b0cf091eff4497ccea33a"}, @@ -2621,12 +3169,48 @@ description = "JSON Matching Expressions" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64"}, {file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"}, ] +[[package]] +name = "joblib" +version = "1.6.0" +description = "Lightweight pipelining with Python functions" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "joblib-1.6.0-py3-none-any.whl", hash = "sha256:3dbbf9f6e4b592a2357b854608e980fe6390d131d7a82f011a377ef2ebef7aba"}, + {file = "joblib-1.6.0.tar.gz", hash = "sha256:2ccc96785b12046c08fd6d55839c12857831b54a3c1673ffadd2f04bfc4eda03"}, +] + +[package.dependencies] +cloudpickle = ">=3.0" + +[package.extras] +docs = ["distributed", "lz4", "matplotlib", "numpy", "numpydoc", "pandas", "psutil", "pydata-sphinx-theme", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx-gallery", "tqdm"] +test = ["distributed", "lz4", "memory_profiler", "numpy", "pytest", "pytest-asyncio", "pytest-cov", "pytest-run-parallel", "pytest-timeout", "threadpoolctl"] + +[[package]] +name = "jsonlines" +version = "4.0.0" +description = "Library with helpers for the jsonlines file format" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "jsonlines-4.0.0-py3-none-any.whl", hash = "sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55"}, + {file = "jsonlines-4.0.0.tar.gz", hash = "sha256:0c6d2c09117550c089995247f605ae4cf77dd1533041d366351f6f298822ea74"}, +] + +[package.dependencies] +attrs = ">=19.2.0" + [[package]] name = "jsonschema" version = "4.26.0" @@ -2634,7 +3218,7 @@ description = "An implementation of JSON Schema validation for Python" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce"}, {file = "jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326"}, @@ -2642,7 +3226,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.3.6" +jsonschema-specifications = ">=2023.03.6" referencing = ">=0.28.4" rpds-py = ">=0.25.0" @@ -2657,7 +3241,7 @@ description = "The JSON Schema meta-schemas and vocabularies, exposed as a Regis optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe"}, {file = "jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d"}, @@ -2666,30 +3250,6 @@ files = [ [package.dependencies] referencing = ">=0.31.0" -[[package]] -name = "kernels" -version = "0.12.1" -description = "Download compute kernels" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"kernels\"" -files = [ - {file = "kernels-0.12.1-py3-none-any.whl", hash = "sha256:4569fe00bc6a2b0f2cccff81841d7917af143c786ade152c3b2a78f61097c757"}, - {file = "kernels-0.12.1.tar.gz", hash = "sha256:41e31ef167add0062cdc302a943ce287194c13b5af12b82a5fa76e7353ca0042"}, -] - -[package.dependencies] -huggingface_hub = ">=0.26.0,<2.0" -packaging = ">=20.0" -pyyaml = ">=6" - -[package.extras] -abi-check = ["kernel-abi-check (>=0.6.2,<0.7.0)"] -benchmark = ["numpy (>=2.0.2)", "requests (>=2.32.5)", "torch"] -docs = ["hf-doc-builder"] -torch = ["torch"] - [[package]] name = "lark" version = "1.2.2" @@ -2697,7 +3257,7 @@ description = "a modern parsing library" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c"}, {file = "lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80"}, @@ -2709,6 +3269,107 @@ interegular = ["interegular (>=0.3.1,<0.4.0)"] nearley = ["js2py"] regex = ["regex"] +[[package]] +name = "latex2sympy2-extended" +version = "1.10.2" +description = "Convert LaTeX math to SymPy expressions" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"eval\" or extra == \"async-rl\"" +files = [ + {file = "latex2sympy2_extended-1.10.2-py3-none-any.whl", hash = "sha256:f910442c5b02a466c1046f47d05cc5285181068b882399281f30102715337fb7"}, + {file = "latex2sympy2_extended-1.10.2.tar.gz", hash = "sha256:41a517ffcc5a140e910a7d1646ce6ff440817e5f9d48fc8279d88bd0925bc389"}, +] + +[package.dependencies] +antlr4-python3-runtime = [ + {version = ">=4.9.3,<=4.13.2"}, + {version = "4.9.3", optional = true, markers = "extra == \"antlr4-9-3\""}, +] +sympy = "*" + +[package.extras] +antlr4-11-0 = ["antlr4-python3-runtime (==4.11.0)"] +antlr4-13-2 = ["antlr4-python3-runtime (==4.13.2)"] +antlr4-9-3 = ["antlr4-python3-runtime (==4.9.3)"] +dev = ["pytest"] + +[[package]] +name = "linkify-it-py" +version = "2.2.0" +description = "Links recognition library with FULL unicode support." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"client\"" +files = [ + {file = "linkify_it_py-2.2.0-py3-none-any.whl", hash = "sha256:3adc40eb5af300b2605fcfdb968c24e1d780a90f1f2221af7c15e5111e94d443"}, + {file = "linkify_it_py-2.2.0.tar.gz", hash = "sha256:907acd2d17ac1fbb9ddb62c8957ccbd6158cac602231a15c3b0cd1e215f03cee"}, +] + +[package.extras] +benchmark = ["pytest", "pytest-benchmark"] +dev = ["black", "flake8", "isort", "pre-commit", "pyproject-flake8"] +doc = ["myst-parser", "sphinx", "sphinx_book_theme"] +test = ["coverage", "pytest", "pytest-cov", "pytest-timeout"] + +[[package]] +name = "litellm" +version = "1.95.0" +description = "Library to easily interface with LLM API providers" +optional = true +python-versions = "<3.15,>=3.10" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "litellm-1.95.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0106b3564b60d00cb5b2810824ebf5071f59c9f9262318884d9c6f040aa2c435"}, + {file = "litellm-1.95.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:053cea1a584bf92d5d44b422f99fa03715bf7c346c8fe080c29bf0c6bd71eddc"}, + {file = "litellm-1.95.0-cp310-cp310-win_amd64.whl", hash = "sha256:667cc7cc58e05a9f9c4bf4588cd4ff5c785fd265060acdfb9147332b75b73ce9"}, + {file = "litellm-1.95.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4bafa3494d503a3c6c1f2eeb785a2af364d85463c71a92cabaaa761c9227d62a"}, + {file = "litellm-1.95.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:4c2a06d2263a07a29228cd3af2b594cf86cd3a4182a7324c517cba88a9415969"}, + {file = "litellm-1.95.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac37bb6d2be191bafc0ce590ed06d21dd7e5a37599ffc1af1071899f6c74b73"}, + {file = "litellm-1.95.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:cb667f84f08520f32b076e03c7a3fa51bf3f7e8b641dade34ab046bf00314d6b"}, + {file = "litellm-1.95.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:1bdf7153557cc0851fa9477b137fde476c56d5de92a5778ecfc6c3a75439a4e1"}, + {file = "litellm-1.95.0-cp312-cp312-win_amd64.whl", hash = "sha256:62cc5d834e8223dbd16c9ad0b46c73354b6d67cc7fa0eba2764ce65b3b8c474f"}, + {file = "litellm-1.95.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:9d80a9adc506bfce48145621d6649e3fd428407811eb00211bcce33344054701"}, + {file = "litellm-1.95.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cf014ff515825ad49937b4cdf95616270789311db7841d16702e0a5b1ac5b067"}, + {file = "litellm-1.95.0-cp313-cp313-win_amd64.whl", hash = "sha256:c73df441153e585832d4e90e3717d17ae888b269daa71d723336369e81ef884b"}, + {file = "litellm-1.95.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:c8e712f95764a9a730f3aec9dff8be916973423411ca12a79a35c8bea3f7d5a4"}, + {file = "litellm-1.95.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:071a63c0e1d949bf7ed5e5c73843cb042a6e00324d3f214b13f8c6b90da6822d"}, + {file = "litellm-1.95.0-cp314-cp314-win_amd64.whl", hash = "sha256:c3684dcf16aefe98bd6f11586d60a9a92bb1bf7e85468a6a4ac28a65532fab3e"}, + {file = "litellm-1.95.0.tar.gz", hash = "sha256:0ef126d52c7a559f8353e50d60fd0d5e7e6c8767ad54df25ddaf79b9edca1afc"}, +] + +[package.dependencies] +aiohttp = ">=3.10,<4.0" +click = ">=8.0.0,<9.0" +fastuuid = ">=0.14.0,<1.0" +httpx = ">=0.28.0,<1.0" +importlib-metadata = ">=8.0.0,<9.0" +jinja2 = ">=3.1.6,<4.0" +jsonschema = ">=4.0.0,<5.0" +openai = ">=2.20.0,<3.0.0" +pydantic = ">=2.10.0,<3.0.0" +python-dotenv = ">=1.0.0,<2.0" +tiktoken = ">=0.8.0,<1.0" +tokenizers = ">=0.21.0,<1.0" + +[package.extras] +bedrock-realtime = ["aws-sdk-bedrock-runtime (>=0.7.0,<0.8.0) ; python_full_version >= \"3.12.0\""] +caching = ["diskcache (>=5.6.3,<6.0)"] +cli = ["inquirerpy (>=0.3.4,<1.0)", "pyyaml (>=6.0.3,<7.0)", "requests (>=2.32.0,<3.0)", "rich (>=13.9.4,<14.0)"] +extra-proxy = ["a2a-sdk (>=1.1.0,<2.0)", "azure-identity (>=1.25.2,<2.0)", "azure-keyvault-secrets (>=4.10.0,<5.0)", "google-cloud-iam (>=2.19.1,<3.0)", "google-cloud-kms (>=2.24.2,<3.0)", "prisma (>=0.11.0,<1.0)", "redisvl (>=0.4.1,<1.0)", "resend (>=2.23.0,<3.0)"] +google = ["google-cloud-aiplatform (>=1.133.0,<2.0)"] +grpc = ["grpcio (==1.78.0)"] +mlflow = ["mlflow (>=3.11.1,<4.0)"] +proxy = ["apscheduler (>=3.11.2,<4.0)", "azure-identity (>=1.25.2,<2.0)", "azure-storage-blob (>=12.28.0,<13.0)", "backoff (>=2.2.1,<3.0)", "boto3 (>=1.43.1,<2.0)", "cryptography (>=48.0.1,<49.0)", "expression (>=5.6.0,<6.0)", "fastapi (>=0.136.3,<1.0)", "fastapi-sso (>=0.19.0,<1.0)", "granian (>=2.7.4,<3.0)", "gunicorn (>=23.0.0,<24.0)", "inquirerpy (>=0.3.4,<1.0)", "litellm-enterprise (==0.1.52)", "litellm-proxy-extras (==0.4.81)", "mcp (>=1.28.1,<2.0)", "orjson (>=3.11.6,<4.0)", "polars (>=1.38.1,<2.0)", "pydantic-settings (>=2.14.1,<3.0)", "pyjwt (>=2.13.0,<3.0)", "pynacl (>=1.6.2,<2.0)", "pyroscope-io (>=0.8.16,<1.0) ; sys_platform != \"win32\"", "python-multipart (>=0.0.27,<1.0)", "pyyaml (>=6.0.3,<7.0)", "restrictedpython (>=8.1,<9.0)", "rich (>=13.9.4,<14.0)", "rq (>=2.7.0,<3.0)", "soundfile (>=0.12.1,<1.0)", "starlette (>=1.0.1,<2.0)", "uvicorn (>=0.33.0,<1.0)", "uvloop (>=0.21.0,<1.0) ; sys_platform != \"win32\"", "websockets (>=15.0.1,<16.0)"] +proxy-runtime = ["anthropic[vertex] (>=0.84.0,<1.0)", "azure-ai-contentsafety (>=1.0.0,<2.0)", "azure-storage-file-datalake (>=12.20.0,<13.0)", "ddtrace (>=4.8.2,<5.0)", "detect-secrets (>=1.5.0,<2.0)", "google-cloud-aiplatform (>=1.133.0,<2.0)", "google-genai (>=1.37.0,<2.0)", "grpcio (==1.78.0)", "langfuse (>=2.59.7,<3.0)", "llm-sandbox (>=0.3.39,<1.0)", "mangum (>=0.17.0,<1.0)", "opentelemetry-api (==1.28.0)", "opentelemetry-exporter-otlp (==1.28.0)", "opentelemetry-instrumentation-fastapi (==0.49b0)", "opentelemetry-sdk (==1.28.0)", "prometheus-client (>=0.20.0,<1.0)", "pypdf (>=6.12.0,<7.0)", "sentry-sdk (>=2.21.0,<3.0)"] +saml = ["python3-saml (>=1.16.0,<2.0)"] +semantic-router = ["aurelio-sdk (>=0.0.19,<1.0) ; python_full_version < \"3.14.0\"", "semantic-router (>=0.1.15,<1.0) ; python_full_version < \"3.14.0\""] +stt-nvidia-riva = ["audioread (>=3.0.1)", "numpy (>=1.26.0)", "nvidia-riva-client (>=2.15.0)", "soundfile (>=0.12.1)"] +utils = ["numpydoc (>=1.8.0,<2.0)"] + [[package]] name = "llguidance" version = "1.3.0" @@ -2716,7 +3377,7 @@ description = "Bindings for the Low-level Guidance (llguidance) Rust library for optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\" and (platform_machine == \"x86_64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\")" +markers = "(extra == \"rl\" or extra == \"async-rl\") and (platform_machine == \"x86_64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\")" files = [ {file = "llguidance-1.3.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f7685222660a762e481ac633d49cc559c64980fe2ee59c8f932a5bb5cbc0c2c2"}, {file = "llguidance-1.3.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:098030ff0687261a3f1bd54cf21fe951fc861d56d37a0671250dd36677eaf224"}, @@ -2735,7 +3396,7 @@ description = "lightweight wrapper around basic LLVM functionality" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "llvmlite-0.44.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:9fbadbfba8422123bab5535b293da1cf72f9f478a65645ecd73e781f962ca614"}, {file = "llvmlite-0.44.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cccf8eb28f24840f2689fb1a45f9c0f7e582dd24e088dcf96e424834af11f791"}, @@ -2767,7 +3428,7 @@ description = "Enforce the output format (JSON Schema, Regex etc) of a language optional = true python-versions = "<4.0,>=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "lm_format_enforcer-0.11.3-py3-none-any.whl", hash = "sha256:cf586350875def1ae7a8fba84fcbbfc8371424b6c9d05c1fcba70aa233fbf06f"}, {file = "lm_format_enforcer-0.11.3.tar.gz", hash = "sha256:e68081c108719cce284a9bcc889709b26ffb085a1945b5eba3a12cfa96d528da"}, @@ -2786,7 +3447,7 @@ description = "Python logging made (stupidly) simple" optional = true python-versions = "<4.0,>=3.5" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c"}, {file = "loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6"}, @@ -2797,7 +3458,200 @@ colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] -dev = ["Sphinx (==8.1.3) ; python_version >= \"3.11\"", "build (==1.2.2) ; python_version >= \"3.11\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.5.0) ; python_version >= \"3.8\"", "mypy (==0.910) ; python_version < \"3.6\"", "mypy (==0.971) ; python_version == \"3.6\"", "mypy (==1.13.0) ; python_version >= \"3.8\"", "mypy (==1.4.1) ; python_version == \"3.7\"", "myst-parser (==4.0.0) ; python_version >= \"3.11\"", "pre-commit (==4.0.1) ; python_version >= \"3.9\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==8.3.2) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==5.0.0) ; python_version == \"3.8\"", "pytest-cov (==6.0.0) ; python_version >= \"3.9\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.1.0) ; python_version >= \"3.8\"", "sphinx-rtd-theme (==3.0.2) ; python_version >= \"3.11\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.23.2) ; python_version >= \"3.8\"", "twine (==6.0.1) ; python_version >= \"3.11\""] +dev = ["Sphinx (==8.1.3) ; python_version >= \"3.11\"", "build (==1.2.2) ; python_version >= \"3.11\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.5.0) ; python_version >= \"3.8\"", "mypy (==v0.910) ; python_version < \"3.6\"", "mypy (==v0.971) ; python_version == \"3.6\"", "mypy (==v1.13.0) ; python_version >= \"3.8\"", "mypy (==v1.4.1) ; python_version == \"3.7\"", "myst-parser (==4.0.0) ; python_version >= \"3.11\"", "pre-commit (==4.0.1) ; python_version >= \"3.9\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==8.3.2) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==5.0.0) ; python_version == \"3.8\"", "pytest-cov (==6.0.0) ; python_version >= \"3.9\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.1.0) ; python_version >= \"3.8\"", "sphinx-rtd-theme (==3.0.2) ; python_version >= \"3.11\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.23.2) ; python_version >= \"3.8\"", "twine (==6.0.1) ; python_version >= \"3.11\""] + +[[package]] +name = "lxml" +version = "6.1.3" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "lxml-6.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:40bcbd9f94166ffe925811e730607385cec959f42fb1bb7dad83748680465221"}, + {file = "lxml-6.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:05f5bce9af14fd1506997594bd81cee6d9c6b58ea80a39c058327aa6371ed9e9"}, + {file = "lxml-6.1.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff88a92cafde90888511242d1c54afcc1a8adbb6dc0a88fa7f87e29e92400d4a"}, + {file = "lxml-6.1.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c00e26288784460885fe76e4d4b293573e0f791f52e6d60e27b42edf005922eb"}, + {file = "lxml-6.1.3-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:773062aec2f2e56b2b22d37054123f0de8a22a4688a0c3376c3fe42685f975cf"}, + {file = "lxml-6.1.3-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6449672f9c93316deb5e2839e18931f468670e44d5bd9b1301a5a9655d45c07"}, + {file = "lxml-6.1.3-cp310-cp310-manylinux_2_28_i686.whl", hash = "sha256:ec295280f4b37769256da025acf5890370355ac589c27e89caae0b5e9eedc702"}, + {file = "lxml-6.1.3-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:5929d9df5e7e3379183be0e21f7d559618a5b61cb63280df6164019242e337ed"}, + {file = "lxml-6.1.3-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6e1eb8a4cbffd5553680ad96be6680e364710656eced73d1dc90ec489df599a3"}, + {file = "lxml-6.1.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:16148acd77ed1d8836a56db883af2f5eed720f9723088110b16a0d08582130a6"}, + {file = "lxml-6.1.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:23c366231259cd75ad06495174701afb3fcb36a92917fa47de2d1f1bd9d95739"}, + {file = "lxml-6.1.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:da85db328e507da922d586c3c7416ec360ec22e9cd9e0700691afacde0c81f53"}, + {file = "lxml-6.1.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0f17d83c48ee9dfd96abae3ac3e2108c76d2fc86ce96355e37b8da9f7f4ecc08"}, + {file = "lxml-6.1.3-cp310-cp310-win32.whl", hash = "sha256:7dd624c1eaa629ad44b59a1a0145fdf2d67895592dce94c9358b938b3d075e65"}, + {file = "lxml-6.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:18a4db52b5a7b53a3540b0b0f4123319334621ee8083d496de314d0bf06ff59a"}, + {file = "lxml-6.1.3-cp310-cp310-win_arm64.whl", hash = "sha256:0feebef8d0521188d0157f758356072e840173aa61ca45b8b3f87959ac283dd5"}, + {file = "lxml-6.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c66f858b82497173f73366795fc6ee8171620e75a338506d6b2e7bc16f5fca11"}, + {file = "lxml-6.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:032a0a97eed428bd143c75a11118238546424ceb2fa311cca5f073aa44658dc4"}, + {file = "lxml-6.1.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4a579dfb9c835f8ab47f4b8ed33440cbc75b806b73297208e6ec2a33e903740b"}, + {file = "lxml-6.1.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:49fbc2682a9306135b7ec49e93f97f9c26689b9b7f96ed2742d8d6497e994d13"}, + {file = "lxml-6.1.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ea2c01cdb16dc12156e455007c406dfaaece0c89aa4ba0e3b47586779f951d41"}, + {file = "lxml-6.1.3-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:527195c188d7d0af748cd48d220ab8cdc5cb99be3d49ac4d9be7324d8abf9bc0"}, + {file = "lxml-6.1.3-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:20384c2bbcbf87180c8c61eb60869699c1ec0cd09b62cfd13804022d860b0867"}, + {file = "lxml-6.1.3-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:424aa5657141d306ba9ad1baab4b2c0a0719040075ee6c66aee9bb2dea2b5054"}, + {file = "lxml-6.1.3-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4736e6c87e603146d8949d8501da621ad20c31015060d3fcf95ace2859f3e3e6"}, + {file = "lxml-6.1.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6374e9e382e5a98c9c5e66d41b357b470da1c54bce30f17f9dc4bcc58436cc1c"}, + {file = "lxml-6.1.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:22eec57e26c418cde02c051ce9914a365e52a7f135a565c6f0480242aeebab48"}, + {file = "lxml-6.1.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8753b8d51dbc86fd335ee31fcf7f3658e9f5c016d4edfb23f76ad295f4b8c9d0"}, + {file = "lxml-6.1.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:207dfc3d47cf0e575e643bbc140dacc8863b39abaa1e5307cd64c7f2365b8a12"}, + {file = "lxml-6.1.3-cp311-cp311-win32.whl", hash = "sha256:18293f8a8d8b6a8e71ef37706b659e3846a4261232158167b1ddf35f6994f633"}, + {file = "lxml-6.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:7ae4949f212a53b007dbc355884fda122545c5764a54256c9217e419a62a6559"}, + {file = "lxml-6.1.3-cp311-cp311-win_arm64.whl", hash = "sha256:2123e5aa075ac20d23c7af489255efd129cbfe190dbe88fd42598cc9df3199b6"}, + {file = "lxml-6.1.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c0710ac085a157b593c38fbcacd950f15c4afa8e2057527185875ab302752bc"}, + {file = "lxml-6.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:623c8799c17128753c65699f1c3aa32402657393a9ad6db09ed8b98ddf76611d"}, + {file = "lxml-6.1.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f683dc6300317700025e41d89a43e0276692ded16113a3c43eab704d605c58e5"}, + {file = "lxml-6.1.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11"}, + {file = "lxml-6.1.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b37772102d44bb6628186accca3a121b1fa3a6b3d97518a8c29a5229ca4c0d0a"}, + {file = "lxml-6.1.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddcf547bea2aee967d6a77779376a45e77e610e8465147a1f3d7e20d539d6e32"}, + {file = "lxml-6.1.3-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c"}, + {file = "lxml-6.1.3-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:a5c18810318303ce9afb3f95e2ddb54834f96fa699a8600433fd5a93dcf44c56"}, + {file = "lxml-6.1.3-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:3e42265103fb385d8642a78672edf376c6f7e1d3598a7a4f9cb1278f2f6b5f6f"}, + {file = "lxml-6.1.3-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:21402998e4b78e7cce237d2788841aaa21ac9a4d1574d04dc2d12ee41ae807b5"}, + {file = "lxml-6.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:38fc4e4e4e084e0bd491949482527d406788045c546d4f8789e93fc527b91385"}, + {file = "lxml-6.1.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5609efdb0d3c95499c00046bc53648b3482ec2175b5503d6e611b3f0555dc71d"}, + {file = "lxml-6.1.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:97ce49699d87ebf8aad631b55d65b33219a4f1bfefbbf5bff19dc9af160aeaf9"}, + {file = "lxml-6.1.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:48542c9acba9ff9450bd18d871d2c2c8787fdb283572b623d206f1b927cd7d9e"}, + {file = "lxml-6.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c55e71a9b1db1f107efb60da49c093689b74c5c31a708e5379e2fd9439d4fbb5"}, + {file = "lxml-6.1.3-cp312-cp312-win32.whl", hash = "sha256:b3ff39654f0ce6ebd4db154211136dbe7e8157bcc3bed2344c87f32c7c6ecb6c"}, + {file = "lxml-6.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:3e9a00d1c2c30936f7add097c41afc5da6556c580909104aafd382cac92a855c"}, + {file = "lxml-6.1.3-cp312-cp312-win_arm64.whl", hash = "sha256:1aeca87830c4fe649dcf93fe2b059525b71c72587f21be4ae4af7103082a79fa"}, + {file = "lxml-6.1.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:3a48093cdb058a93af842ede9703520e810b05dcd0fc6d7190a06376c3bfb6bd"}, + {file = "lxml-6.1.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:887c021d9a977cff89cb273047c1352997b772a8908a25c21836861f69b92be1"}, + {file = "lxml-6.1.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:611a51e61c92f62345a50b0035df6fc0d678f9299f33728826d831598862f59d"}, + {file = "lxml-6.1.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b477912f42c5c33405a10c759d22f80cf5af043ae02d95b9d8e5e5bc555739ed"}, + {file = "lxml-6.1.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cffe18571ccc51d742cd08cbb3f8b756de9311d18c7ea98f5d92f37b8fb60c2"}, + {file = "lxml-6.1.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:75cc6569e86be5785b6188ef1642670c6adbc984e81ec35e224842ecd9eefcc8"}, + {file = "lxml-6.1.3-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d85dfab42dd672f87a7f76e9de7172962aee69fa12044f0d6e1a23cbd53fb80e"}, + {file = "lxml-6.1.3-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:42632b4024ab24a6b488f559ac851312509888b6b80ae2aa11cf29a646a0d245"}, + {file = "lxml-6.1.3-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:febd35ef45f603c2d74b74655efdbf45e14f55fc0aef4ac82b663ca829b283e0"}, + {file = "lxml-6.1.3-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a43b3bdf11e477dc7770609d3477316f974354dfc8425d596f64f471cc8daf6e"}, + {file = "lxml-6.1.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d582042c69857c364e8153de6e18e0da9b7b515a6a8113caf69a6ec8e0520f2"}, + {file = "lxml-6.1.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8e49a646acfab83c68974f4aa1d0a2acca9e88d7d627ae0fc13201b14b76d310"}, + {file = "lxml-6.1.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0dee106e9aa97fb00541b1ed7827070564d0549c3d3fba8920e6b20fd980f748"}, + {file = "lxml-6.1.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:dd5e90f34cffcfed97f36cf066325773d2b6021c60c29942e53a18b028501b1d"}, + {file = "lxml-6.1.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d9b3e7d71bf6acff341233417abbdface29c647e3113892d9aaedc02eb4aa2bc"}, + {file = "lxml-6.1.3-cp313-cp313-win32.whl", hash = "sha256:160fcf381f76c3aeac28a756bec44f48942a8f7245a87aa28e3a523b4d90cd87"}, + {file = "lxml-6.1.3-cp313-cp313-win_amd64.whl", hash = "sha256:e477aca0bc0d19f3b4ae9e4f2a1cfd687c31bf772d78734910658186b40b2477"}, + {file = "lxml-6.1.3-cp313-cp313-win_arm64.whl", hash = "sha256:b1cc980905221a5d8b3c476330730b3adb40ff80add71ffbdb6215ba055656f1"}, + {file = "lxml-6.1.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2bec13085dc8ef48a3fe62f7dfcacfeda2c785cdf19cc8eeda2bb9ed081da165"}, + {file = "lxml-6.1.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4f4db7c7e954d289d71878938348b3d91b904a3e8210a11939359fb758a58e7d"}, + {file = "lxml-6.1.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2cae5d5c90a62d9139c512a0cb1aad1d182b022b5740daea2617eb5bf7fc658e"}, + {file = "lxml-6.1.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c6c0c13128a32eb04a51357e56a094e13aa8e6d3d1884de2e9ae923f6915e1a8"}, + {file = "lxml-6.1.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2221e88679d1351e9a40aaee54bc65679b9795bbd0160bc3d5e36b163344eb75"}, + {file = "lxml-6.1.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cfb398886a7eb4c719161c3efcff2a1248febc53a4d8e5072d2d8a87fed84ac9"}, + {file = "lxml-6.1.3-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7eb78ba28b187e1e9203a55c60fcf70df2d22cb205fe6d51b9383d6097419f0"}, + {file = "lxml-6.1.3-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:ea6b1e9105b4b24a34c722432d9fb578f9ed83af21fa1abda639011e0f22bbb6"}, + {file = "lxml-6.1.3-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:e8b17e23df3e827a69d25af70990ca2420e92668aaffaeeb3cd2351d7916a023"}, + {file = "lxml-6.1.3-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b7c37339d7e75cab9a123a04248e243cefefb302ad6db566ea0c77cbcde421e"}, + {file = "lxml-6.1.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:83e3a51e7933db700a0da0db31849db3a24022d9970da9bb73001e1d0326fd92"}, + {file = "lxml-6.1.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9bde9ae026a55b9a192078dfa6e27dd0ca4a050171ab6272e92f97b757dfdf48"}, + {file = "lxml-6.1.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1a635e837b50a1819bebfedaac5916498ea024120969da8790500148fb0a894d"}, + {file = "lxml-6.1.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d0c5c362bc94f1929dc7e96e715bbe7bd17037f802e6d8f0d1545df9133c0559"}, + {file = "lxml-6.1.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c59e4265608da6a041f54646ecc0c9ecdbb19aaf14c4c684bb6c2114998cc415"}, + {file = "lxml-6.1.3-cp314-cp314-win32.whl", hash = "sha256:2e62c569ec7531b679b184cbfe335c501c1d13c4b363560013019962eb630e6d"}, + {file = "lxml-6.1.3-cp314-cp314-win_amd64.whl", hash = "sha256:66299564c046bc7e0cc5de5106601eae907e9fa5904cd68a323380a8502f7861"}, + {file = "lxml-6.1.3-cp314-cp314-win_arm64.whl", hash = "sha256:ebd054ad1737a68fb7c5c073d405cef2b88bb824e294de3b4a4e995b47f0e376"}, + {file = "lxml-6.1.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5a143e6207579de8baeded4eaac9134413200359f1969d636f0bfb98ee8c3c8f"}, + {file = "lxml-6.1.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a1cec0f99b9b914d39176347a93b7610dc09324491aee1cbc57cd291a41a1d55"}, + {file = "lxml-6.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f6b9d2aad499c769ee8287609ab0e6de99d8bcea99c6e6c2e64945259fd52fb2"}, + {file = "lxml-6.1.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a23fefdb345b2d4d0ff2860571b5ff9a89a28b6a120f720e8fb0324d346626"}, + {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:545ccc14fb05485f48b4439ec35beb16d5b5280eb6c81c658bd4707a2a119414"}, + {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:93476b6514b373fc6ca67d26c442784f7807c86f00635bfe79f935c3eab2af17"}, + {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8db38ff3fb7aee7d6a82ae4da2eef1178656fe1216841fbd24870062a9d60473"}, + {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:25f4118c438f96bb466e83108506d03d5c31b1bd2387e83e5b070bda6ded9c37"}, + {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:1beb0f9909b26cee938df9ba56b15252a84429b1fc30ce6fca161390b9789a70"}, + {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3a27ac6c780c8b8a1cd231b58407634cafc1c4cc28cd6c7141362df0f36351e7"}, + {file = "lxml-6.1.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a1932d7ce78a561367512c594fe66eac2b2ec9b9264cfd9b5f950622f4a116e2"}, + {file = "lxml-6.1.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7d0f5976aa2701996f759b30172925829867547bb073af0ae67d1307a0f0262c"}, + {file = "lxml-6.1.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e7ce578aa8a80910a72a8ca0bbea3baae10100827249001999726a788456d8"}, + {file = "lxml-6.1.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d97c5227621af74b111882a290b10f371780a38eef9d9e730408fba2259b52fb"}, + {file = "lxml-6.1.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da707f14ea3c35ee463d50acd596d6488e4b2b4ae7cf77a5bf93f55c023d63e8"}, + {file = "lxml-6.1.3-cp314-cp314t-win32.whl", hash = "sha256:9efe56a68179f3adc4de41861c9358931db03837c48dd5e1c78077b84dd07f3a"}, + {file = "lxml-6.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c9389b3784b56c58d933b5e0aecdf28f901b073ff385358d8a7d40907f6e14b2"}, + {file = "lxml-6.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32a409be3190b088f960ac92bfedfbef2f86c49ff940765e1548177592d20026"}, + {file = "lxml-6.1.3-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:6ea2f13dce778ca072ccee598bca46a092ce192e8fd907b6c1f0e52c800529a0"}, + {file = "lxml-6.1.3-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c581b1d68b3845fb86c6b2983e755b29bf001461c59fa411d2c26a911b6559a9"}, + {file = "lxml-6.1.3-cp315-cp315-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e01125896585139453cab8cb235893644d8815d7509520da95ae3ee8d1c1f79"}, + {file = "lxml-6.1.3-cp315-cp315-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:290f66b97ede0e552e1cb44a0fd8a74f9753ee635b50830a0b122fb72788d015"}, + {file = "lxml-6.1.3-cp315-cp315-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73fc05988ed20809450474ba760a87c8ad4e455fc09783c02195e56ec634b41a"}, + {file = "lxml-6.1.3-cp315-cp315-manylinux_2_31_armv7l.whl", hash = "sha256:dc3a44689eea43eab836e5c98a8ab015dc2419987d1ea6eafc7c590cdff86bed"}, + {file = "lxml-6.1.3-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:209c3ccbfe35a04ac6d24f0611f9d1cbf8025d49991b14acd935236234d6c156"}, + {file = "lxml-6.1.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2f5b2a2b9811b853b39bfa41367c6d78747b8e3e80e07fc5a24aae295c1a4d7d"}, + {file = "lxml-6.1.3-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:6a406d0b3cb207b0fa460ed4dc93e866f44f105da0169361cb18ff998a44c7f0"}, + {file = "lxml-6.1.3-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:53258656846f5c48996b882fb4b135885e088a3ad3d96b4bc0530f95124d1f69"}, + {file = "lxml-6.1.3-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:aa633613ff907ea91b9b0489a1f0da1b8725d8c6ccec6b77e8a1c9c235044bb0"}, + {file = "lxml-6.1.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:90f709b9accab6b2e4d14f5c8718203877a0486bcb3afd74d8b539ecd1e961d4"}, + {file = "lxml-6.1.3-cp315-cp315-win32.whl", hash = "sha256:b4fc6b03b9d9d90557274f571ab30e7fbbfc527955536935d96f98b6817a86e4"}, + {file = "lxml-6.1.3-cp315-cp315-win_amd64.whl", hash = "sha256:33cadd956b667997e4de1635fce9541f2e8ede2038fcde8cf55aa14d571d1bad"}, + {file = "lxml-6.1.3-cp315-cp315-win_arm64.whl", hash = "sha256:8a330c0ee5fa318c7b5cbbaad882baeca3f570357e7eb25ab34bf31008150758"}, + {file = "lxml-6.1.3-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:0bf5a3e397df2ec4258eb5eea4c1ac6cf013ca1abd04a176903bff20a70021fe"}, + {file = "lxml-6.1.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:13d22c0d57355366b393936acf6b98a5e0edeadddd3fccbc6a846c50a76b8741"}, + {file = "lxml-6.1.3-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cad7617727a96d189bd6f979d0fadf765198c7934e85f4edaba9bf3ad919a300"}, + {file = "lxml-6.1.3-cp315-cp315t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cae82b5ca24b0c2beedb269f6e2a96f466acd926879ab00ae19f1a65cbf9ffb0"}, + {file = "lxml-6.1.3-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:69cafd61aea04ebb3502c93c2aaa568b12931ca0802231e0b5de76bf8b6e74bd"}, + {file = "lxml-6.1.3-cp315-cp315t-manylinux_2_31_armv7l.whl", hash = "sha256:dc205732d593118cf701d986f40e9de7801bb2e371cb189ddbda9b7348f4d97e"}, + {file = "lxml-6.1.3-cp315-cp315t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88e719b9437f148f7e1465df845c758dd1598618cbea3a2fd1e61a715542f2b2"}, + {file = "lxml-6.1.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:40983eabefd13da003e68170928c7acc011f0d095eefce5871a3c71c9385fb9a"}, + {file = "lxml-6.1.3-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:fad67b12ffe0f71e02b4932b04883cbc76a9072bbd30731409d3523cf058b011"}, + {file = "lxml-6.1.3-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6cd11e7550d89e551a87dcec30f04b1fca32e86b68708aa01a4daa455d8605e5"}, + {file = "lxml-6.1.3-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ca0ec532ad2f5ba1e5ec120ac157769c57f01855b3d8bf37213f5d88abd9ba0a"}, + {file = "lxml-6.1.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e99e09ab7741f1281e2677f4c0058c7f5267d182530b09c87e4f6aa26adf3887"}, + {file = "lxml-6.1.3-cp315-cp315t-win32.whl", hash = "sha256:ace1d2c83b2bd24db5940600541140e87a325e119cb32d5fa9ad720d7e76648e"}, + {file = "lxml-6.1.3-cp315-cp315t-win_amd64.whl", hash = "sha256:b49638355ea3bebba70da783ccbc630fd72afa16bc46c54474bfa1f9a915bbc6"}, + {file = "lxml-6.1.3-cp315-cp315t-win_arm64.whl", hash = "sha256:5a721a98c649855963811b59b55755b30566e7f7fc40bdc9803d66dee9f811cf"}, + {file = "lxml-6.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:13a620a3fcc20023f9e6ed5c383e00e826f1c2d5db554df2f67240760f9118e8"}, + {file = "lxml-6.1.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fbfb70ba01355251faf6b293171df49f73a88a1b6494db109ffea85442574458"}, + {file = "lxml-6.1.3-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:302f72413251c03f671e063c9414bed5dc8c927069e5abb69245521e51a4e81b"}, + {file = "lxml-6.1.3-cp38-cp38-manylinux_2_28_i686.whl", hash = "sha256:ce1f220114959941170e22b8ad44279f6dee2dcef7591814d01ae805dc058889"}, + {file = "lxml-6.1.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:170773d8a3cdc76259065523ddd978c44f9806e28605f08812e8f86783e44ac6"}, + {file = "lxml-6.1.3-cp38-cp38-win32.whl", hash = "sha256:92d96586376fb79a33474797186bf993250152ee5c32650b67db78d54b92e6f3"}, + {file = "lxml-6.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:d44442effeb8781f392340c5dc8c6716fba41dbeacb82fd4c0f09026fb5ff682"}, + {file = "lxml-6.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:869dfcd4d381cb0ea87085cc4f011b9171b494ef21e76ad8665f6d5e2d1dc8a1"}, + {file = "lxml-6.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6ba4fe5bfbef6811a8e49b3719cde373ad399006c0c1ac184b7297116ecbba5d"}, + {file = "lxml-6.1.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:61116cec57ed69aebc70f37a545eec095339bb829efbdabcfb97c51e9536e158"}, + {file = "lxml-6.1.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e11e885e0704be185867fcf71b904d8f65d7d6877bc121f69870b0d0479ba7b"}, + {file = "lxml-6.1.3-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41e2d428110b408e963b6fb18f9bbf1f5c027b56bd4b498d54556476c0aeb1c3"}, + {file = "lxml-6.1.3-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa9fd1ee2a5dacfc41039ed49ffeeacfa75bafbd255b69f3b578e11897a0e623"}, + {file = "lxml-6.1.3-cp39-cp39-manylinux_2_28_i686.whl", hash = "sha256:7f75b9b9fec2a9c6b18095c81865580e795b1441c429e42d22fcc82a77f40039"}, + {file = "lxml-6.1.3-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:cc669256d28736f7f3a149df5c380c50ace2692ba3e62203d10656fade4a2145"}, + {file = "lxml-6.1.3-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d077f21f4b16f0471353883748f126f62038760397c107bb9fad2ca94dc0dfb7"}, + {file = "lxml-6.1.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0d12846d6ce434fb3857918eef4315ec9b4769deb020c75828798614bfcfd"}, + {file = "lxml-6.1.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:2b9b1325ca1c2a9a2dbb6eb913ae563313f2082ae60b03210f7e83ee80712274"}, + {file = "lxml-6.1.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:a2e3f70673a1d5b82f38255f777d26cd855bf2092b1436c4867464a7892f9238"}, + {file = "lxml-6.1.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c34ca1dc41bd86d9ff830d5bdf4e4a752bba6c54f7d2707027ce0eabd36084c9"}, + {file = "lxml-6.1.3-cp39-cp39-win32.whl", hash = "sha256:b50343241eb69fd85f7791cf8bcc7b1c4729826b7d59ba2f6b27db29638fa745"}, + {file = "lxml-6.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:0794e04ba343852c6d78e996c58ef4b8e579b4ecc72f8df0d4058bf843b4c96e"}, + {file = "lxml-6.1.3-cp39-cp39-win_arm64.whl", hash = "sha256:0ab2467e405e748d93495fb5568e74044802b8d3ff2b2a1607c3f78c6e982de5"}, + {file = "lxml-6.1.3-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4b061064b4a2fe8598a466d723d43dbcd5a610a5d5cfe02fb6226f5c17349f75"}, + {file = "lxml-6.1.3-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8499d464de86fab0f102313cce32a9bed9ab1f06ec813cf025cb790964fbb765"}, + {file = "lxml-6.1.3-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e67324961ac9bbe616cce5100514d2e34d88665aeb07071e8b16eac55d06d94"}, + {file = "lxml-6.1.3-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d12669a2c419b0e8dc423d23dea24bb82f6f9cb829f32e04674b0ba40322a7c"}, + {file = "lxml-6.1.3-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:97acecb11cbc411473f15b8d780df06d7a9f3a2aad9aca78364f56640c8fb70e"}, + {file = "lxml-6.1.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f8b9c8ceebae6387d0dc77f7f4dbbfbfc962dba2efbfe6877486075a480726b4"}, + {file = "lxml-6.1.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d2765c18ce303149ee804b1f3dad11232726dd0a702d73a15cf19179ac8cc962"}, + {file = "lxml-6.1.3-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d5a748d12dd9b535e0a130f60dae9ddf0adafbabe61e7864f55c7436c84547a"}, + {file = "lxml-6.1.3-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:41096ec0740a58dad03d3ae0c7486d306d20becefb13ceb1649835ab3eb64167"}, + {file = "lxml-6.1.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:415e3a115c0d510e329020012834d1c0aa1c581ee53a218603e38abbc1dea70a"}, + {file = "lxml-6.1.3-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20428910dae17a1a93152a3ff2c0441d2f4932992c0797d65651dd0561f1792f"}, + {file = "lxml-6.1.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bc8dd3d9c93e70c3df974a201ac2958b6d77b465d813c51d1f15fa8e645763ae"}, + {file = "lxml-6.1.3-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3847e71a78cbbc1aff955dbbbaf2fff12153f611d3162c5beaa3395636cbc2f9"}, + {file = "lxml-6.1.3-pp39-pypy39_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe91993149523aa59941b9e3c90e2eb45f57ad014697aef6c8b13339a59c019e"}, + {file = "lxml-6.1.3-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71532ebf30be0048a45559b4fab15333fbaaf9042f658e878d918ecd0cf09805"}, + {file = "lxml-6.1.3-pp39-pypy39_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1b50797ac246bb2942a04b6c0f69af0667aba7cf7535f39bbb1b3208fd5d128"}, + {file = "lxml-6.1.3-pp39-pypy39_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2bb7d703bed7ac893bf7f40d97b5d9279d35d2ce460624ca28929eab0d5a3d"}, + {file = "lxml-6.1.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:be5346653c0b0e34be96869ff9dbeba23860156f89a2896a64c64fb419260cb6"}, + {file = "lxml-6.1.3.tar.gz", hash = "sha256:45222d94ddd511536f3b2f7d9deae3b2339b4ce0f075f1ca25703b07cad9dd21"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html-clean = ["lxml_html_clean"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] [[package]] name = "markdown" @@ -2806,7 +3660,7 @@ description = "Python implementation of John Gruber's Markdown." optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"docs\"" +markers = "extra == \"eval\" or extra == \"docs\"" files = [ {file = "markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36"}, {file = "markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950"}, @@ -2820,16 +3674,16 @@ testing = ["coverage", "pyyaml"] name = "markdown-it-py" version = "2.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" -optional = true +optional = false python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"tinker\" or extra == \"docs\"" files = [ {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, ] [package.dependencies] +linkify-it-py = {version = ">=1,<3", optional = true, markers = "extra == \"linkify\""} mdurl = ">=0.1,<1.0" [package.extras] @@ -2941,6 +3795,52 @@ files = [ {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, ] +[[package]] +name = "math-verify" +version = "0.8.0" +description = "HuggingFace library for verifying mathematical answers" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "math_verify-0.8.0-py3-none-any.whl", hash = "sha256:31ca651296d817a9bb3fd58ca1fd0d192dcea709b1e5ecf2d0a4514c16f89087"}, + {file = "math_verify-0.8.0.tar.gz", hash = "sha256:3295e0adb94bfe553ff6e3189c44f1916a85aa24ab5d1900f2086a706e28f7c4"}, +] + +[package.dependencies] +latex2sympy2_extended = "1.10.2" + +[package.extras] +antlr4-11-0 = ["latex2sympy2_extended[antlr4-11-0]"] +antlr4-13-2 = ["latex2sympy2_extended[antlr4-13-2]"] +antlr4-9-3 = ["latex2sympy2_extended[antlr4-9-3]"] +dev = ["math-verify[format]", "math-verify[test]"] +format = ["ruff"] +inference = ["lighteval[math]"] +test = ["pytest"] + +[[package]] +name = "mcore-bridge" +version = "1.6.4" +description = "MCore-Bridge: Making Megatron training as simple as Transformers" +optional = true +python-versions = ">=3.8.0" +groups = ["main"] +markers = "extra == \"megatron\"" +files = [ + {file = "mcore_bridge-1.6.4-py3-none-any.whl", hash = "sha256:df8ab70f3a8122184833221af8f7756ff937bf980ee6fef6cbeaa3fcc1d00a73"}, + {file = "mcore_bridge-1.6.4.tar.gz", hash = "sha256:6df211f0e810d0d3d4be61de72bf68aa730c7da00a7ed5ecc18ffd1ec3563cf2"}, +] + +[package.dependencies] +megatron-core = ">=0.16,<0.20" +modelscope = "*" +peft = ">=0.11,<0.21" +safetensors = "*" +tqdm = "*" +transformers = ">=4.33,<5.17.0" + [[package]] name = "mcp" version = "1.26.0" @@ -2948,7 +3848,7 @@ description = "Model Context Protocol SDK" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "mcp-1.26.0-py3-none-any.whl", hash = "sha256:904a21c33c25aa98ddbeb47273033c435e595bbacfdb177f4bd87f6dceebe1ca"}, {file = "mcp-1.26.0.tar.gz", hash = "sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66"}, @@ -2982,7 +3882,7 @@ description = "Collection of plugins for markdown-it-py" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"docs\"" +markers = "extra == \"client\" or extra == \"docs\"" files = [ {file = "mdit-py-plugins-0.3.5.tar.gz", hash = "sha256:eee0adc7195e5827e17e02d2a258a2ba159944a0748f59c5099a4a27f78fcf6a"}, {file = "mdit_py_plugins-0.3.5-py3-none-any.whl", hash = "sha256:ca9a0714ea59a24b2b044a1831f48d817dd0c817e84339f20e7889f392d77c4e"}, @@ -3000,10 +3900,9 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -optional = true +optional = false python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"tinker\" or extra == \"docs\"" files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -3046,7 +3945,7 @@ description = "Mistral-common is a library of common utilities for Mistral AI." optional = true python-versions = "<3.14,>=3.10.0" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "mistral_common-1.9.1-py3-none-any.whl", hash = "sha256:9e2b2520b6f67bac2e2bb06fcf985b7a1277b01938da2b7cda8cf0fdbfa92e91"}, {file = "mistral_common-1.9.1.tar.gz", hash = "sha256:550583d70a395c3586cfb748ffab53bd1d7c3409507f0efc0118bff30ffb26e9"}, @@ -3126,6 +4025,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.23.3", markers = "python_version >= \"3.11\""}, + {version = ">=1.21.2", markers = "python_version == \"3.10\""}, + {version = ">=2.1.0", markers = "python_version >= \"3.13\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] @@ -3139,7 +4040,7 @@ description = "A framework for machine learning on Apple silicon." optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\" and platform_machine == \"arm64\" and platform_system == \"Darwin\"" +markers = "(extra == \"rl\" or extra == \"async-rl\") and platform_machine == \"arm64\" and platform_system == \"Darwin\"" files = [ {file = "mlx-0.31.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:38680838e0dd9a621ed4adc5a9ed8b94aeb6a4798142fbe215b821b8c6b8fc36"}, {file = "mlx-0.31.0-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:7aded590bcf6839307c3acc899e196936991f97b499ddbdd0cd3b228bf10792f"}, @@ -3185,7 +4086,7 @@ description = "LLMs with MLX and the Hugging Face Hub" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\" and platform_machine == \"arm64\" and platform_system == \"Darwin\"" +markers = "(extra == \"rl\" or extra == \"async-rl\") and platform_machine == \"arm64\" and platform_system == \"Darwin\"" files = [ {file = "mlx_lm-0.29.1-py3-none-any.whl", hash = "sha256:440941b3054c2a2216e97615de584cc90fa1ea874782e20699b9895721fad8dc"}, {file = "mlx_lm-0.29.1.tar.gz", hash = "sha256:b99180d8f33d33a077b814e550bfb2d8a59ae003d668fd1f4b3fff62a381d34b"}, @@ -3214,7 +4115,7 @@ description = "A framework for machine learning on Apple silicon." optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\" and platform_machine == \"arm64\" and platform_system == \"Darwin\"" +markers = "(extra == \"rl\" or extra == \"async-rl\") and platform_machine == \"arm64\" and platform_system == \"Darwin\"" files = [ {file = "mlx_metal-0.31.0-py3-none-macosx_14_0_arm64.whl", hash = "sha256:1c572a6e3634a63060c103b0c38ac309e2d217be15519e3d8f0d6b452bb015f5"}, {file = "mlx_metal-0.31.0-py3-none-macosx_15_0_arm64.whl", hash = "sha256:554dc7cb29e0ea5fb6941df42f11a1de385b095848e6183c7a99d7c1f1a11f5d"}, @@ -3228,7 +4129,7 @@ description = "Python toolkit for standardized model hosting container implement optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "model_hosting_container_standards-0.1.13-py3-none-any.whl", hash = "sha256:be307d4a988cc660df4e6bd8bdedb7917844bac940e332f9fd001cb385d7994c"}, {file = "model_hosting_container_standards-0.1.13.tar.gz", hash = "sha256:27a1333410dde2719286a300a2803e24fdde407baa91894eb845c0f268aa194d"}, @@ -3256,19 +4157,47 @@ files = [ ] [package.dependencies] -addict = {version = "*", optional = true, markers = "extra == \"framework\""} -attrs = {version = "*", optional = true, markers = "extra == \"framework\""} -datasets = {version = ">=3.0.0,<=3.6.0", optional = true, markers = "extra == \"framework\""} -einops = {version = "*", optional = true, markers = "extra == \"framework\""} +addict = [ + {version = "*", optional = true, markers = "extra == \"framework\""}, + {version = "*", optional = true, markers = "extra == \"datasets\""}, +] +attrs = [ + {version = "*", optional = true, markers = "extra == \"framework\""}, + {version = "*", optional = true, markers = "extra == \"datasets\""}, +] +datasets = [ + {version = ">=3.0.0,<=3.6.0", optional = true, markers = "extra == \"framework\""}, + {version = ">=3.0.0,<=3.6.0", optional = true, markers = "extra == \"datasets\""}, +] +einops = [ + {version = "*", optional = true, markers = "extra == \"framework\""}, + {version = "*", optional = true, markers = "extra == \"datasets\""}, +] filelock = "*" -Pillow = {version = "*", optional = true, markers = "extra == \"framework\""} -python-dateutil = {version = ">=2.1", optional = true, markers = "extra == \"framework\""} +oss2 = {version = "*", optional = true, markers = "extra == \"datasets\""} +Pillow = [ + {version = "*", optional = true, markers = "extra == \"framework\""}, + {version = "*", optional = true, markers = "extra == \"datasets\""}, +] +python-dateutil = [ + {version = ">=2.1", optional = true, markers = "extra == \"framework\""}, + {version = ">=2.1", optional = true, markers = "extra == \"datasets\""}, +] PyYAML = {version = ">=5.4", optional = true, markers = "extra == \"framework\""} requests = ">=2.25" -scipy = {version = "*", optional = true, markers = "extra == \"framework\""} +scipy = [ + {version = "*", optional = true, markers = "extra == \"framework\""}, + {version = "*", optional = true, markers = "extra == \"datasets\""}, +] setuptools = "*" -simplejson = {version = ">=3.3.0", optional = true, markers = "extra == \"framework\""} -sortedcontainers = {version = ">=1.5.9", optional = true, markers = "extra == \"framework\""} +simplejson = [ + {version = ">=3.3.0", optional = true, markers = "extra == \"framework\""}, + {version = ">=3.3.0", optional = true, markers = "extra == \"datasets\""}, +] +sortedcontainers = [ + {version = ">=1.5.9", optional = true, markers = "extra == \"framework\""}, + {version = ">=1.5.9", optional = true, markers = "extra == \"datasets\""}, +] tqdm = ">=4.64.0" transformers = {version = "*", optional = true, markers = "extra == \"framework\""} urllib3 = ">=1.26" @@ -3292,6 +4221,19 @@ science = ["Pillow", "PyYAML (>=5.4)", "addict", "attrs", "biopython", "datasets server = ["fastapi", "sse-starlette", "uvicorn"] tests = ["expecttest", "flake8", "isort (>=4.3.21)", "pre-commit", "yapf (==0.30.0)"] +[[package]] +name = "more-itertools" +version = "11.1.0" +description = "More routines for operating on iterables, beyond itertools" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192"}, + {file = "more_itertools-11.1.0.tar.gz", hash = "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d"}, +] + [[package]] name = "mpmath" version = "1.3.0" @@ -3317,7 +4259,7 @@ description = "MessagePack serializer" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "msgpack-1.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2"}, {file = "msgpack-1.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87"}, @@ -3390,7 +4332,7 @@ description = "A fast serialization and validation library, with builtin support optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "msgspec-0.20.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:23a6ec2a3b5038c233b04740a545856a068bc5cb8db184ff493a58e08c994fbf"}, {file = "msgspec-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cde2c41ed3eaaef6146365cb0d69580078a19f974c6cb8165cc5dcd5734f573e"}, @@ -3611,6 +4553,9 @@ files = [ {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, ] +[package.dependencies] +typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} + [[package]] name = "multiprocess" version = "0.70.16" @@ -3660,10 +4605,58 @@ sphinx = ">=5,<7" [package.extras] code-style = ["pre-commit (>=3.0,<4.0)"] linkify = ["linkify-it-py (>=1.0,<2.0)"] -rtd = ["ipython", "pydata-sphinx-theme (==0.13.0rc4)", "sphinx-autodoc2 (>=0.4.2,<0.5.0)", "sphinx-book-theme (==1.0.0rc2)", "sphinx-copybutton", "sphinx-design2", "sphinx-pyscript", "sphinx-tippy (>=0.3.1)", "sphinx-togglebutton", "sphinxext-opengraph (>=0.7.5,<0.8.0)", "sphinxext-rediraffe (>=0.2.7,<0.3.0)"] +rtd = ["ipython", "pydata-sphinx-theme (==v0.13.0rc4)", "sphinx-autodoc2 (>=0.4.2,<0.5.0)", "sphinx-book-theme (==1.0.0rc2)", "sphinx-copybutton", "sphinx-design2", "sphinx-pyscript", "sphinx-tippy (>=0.3.1)", "sphinx-togglebutton", "sphinxext-opengraph (>=0.7.5,<0.8.0)", "sphinxext-rediraffe (>=0.2.7,<0.3.0)"] testing = ["beautifulsoup4", "coverage[toml]", "pytest (>=7,<8)", "pytest-cov", "pytest-param-files (>=0.3.4,<0.4.0)", "pytest-regressions", "sphinx-pytest"] testing-docutils = ["pygments", "pytest (>=7,<8)", "pytest-param-files (>=0.3.4,<0.4.0)"] +[[package]] +name = "narwhals" +version = "2.26.0" +description = "Extremely lightweight compatibility layer between dataframe libraries" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "narwhals-2.26.0-py3-none-any.whl", hash = "sha256:29326d74f107c347fd1009bd58e38d9f7c7c5b51e6de97bc93dbc325d9038b54"}, + {file = "narwhals-2.26.0.tar.gz", hash = "sha256:6b9cadca82f375c7e4cf584fdc86ca25da54827307a9c58f94547ee6104b82dd"}, +] + +[package.extras] +cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] +dask = ["dask[dataframe] (>=2024.8)"] +duckdb = ["duckdb (>=1.1)"] +ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)"] +modin = ["modin (>=0.22.0)"] +pandas = ["pandas (>=1.3.4)"] +polars = ["polars (>=0.20.4)"] +pyarrow = ["pyarrow (>=13.0.0)"] +pyspark = ["pyspark (>=3.5.0)"] +pyspark-connect = ["pyspark[connect] (>=3.5.0)"] +sql = ["narwhals[duckdb]", "sqlparse (>=0.5.5)"] +sqlframe = ["sqlframe (>=3.22.0,!=3.39.3)"] + +[[package]] +name = "networkx" +version = "3.4.2" +description = "Python package for creating and manipulating graphs and networks" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version == \"3.10\"" +files = [ + {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, + {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, +] + +[package.extras] +default = ["matplotlib (>=3.7)", "numpy (>=1.24)", "pandas (>=2.0)", "scipy (>=1.10,!=1.11.0,!=1.11.1)"] +developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] +doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.15)", "sphinx (>=7.3)", "sphinx-gallery (>=0.16)", "texext (>=0.6.7)"] +example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "momepy (>=0.7.2)", "osmnx (>=1.9)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] +extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] +test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] + [[package]] name = "networkx" version = "3.6.1" @@ -3671,6 +4664,7 @@ description = "Python package for creating and manipulating graphs and networks" optional = false python-versions = "!=3.14.1,>=3.11" groups = ["main"] +markers = "python_version >= \"3.11\"" files = [ {file = "networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762"}, {file = "networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509"}, @@ -3694,7 +4688,7 @@ description = "Ninja is a small build system with a focus on speed" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "ninja-1.13.0-py3-none-macosx_10_9_universal2.whl", hash = "sha256:fa2a8bfc62e31b08f83127d1613d10821775a0eb334197154c4d6067b7068ff1"}, {file = "ninja-1.13.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3d00c692fb717fd511abeb44b8c5d00340c36938c12d6538ba989fe764e79630"}, @@ -3717,6 +4711,34 @@ files = [ {file = "ninja-1.13.0.tar.gz", hash = "sha256:4a40ce995ded54d9dc24f8ea37ff3bf62ad192b547f6c7126e7e25045e76f978"}, ] +[[package]] +name = "nltk" +version = "3.10.3" +description = "Natural Language Toolkit" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "nltk-3.10.3-py3-none-any.whl", hash = "sha256:ff9598a8e20518ee0d557745890cc4435b9578489e2dcbc69c4f81fa060caf7c"}, + {file = "nltk-3.10.3.tar.gz", hash = "sha256:bb9327a461c3811c2fa4900e03840401f2126adfb30c0072827c433bd2444ea4"}, +] + +[package.dependencies] +click = "*" +defusedxml = "*" +joblib = "*" +regex = ">=2021.8.3" +tqdm = "*" + +[package.extras] +all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] +corenlp = ["requests"] +machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] +plot = ["matplotlib"] +tgrep = ["pyparsing"] +twitter = ["twython"] + [[package]] name = "numba" version = "0.61.2" @@ -3724,7 +4746,7 @@ description = "compiling Python code using LLVM" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "numba-0.61.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:cf9f9fc00d6eca0c23fc840817ce9f439b9f03c8f03d6246c0e7f0cb15b7162a"}, {file = "numba-0.61.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ea0247617edcb5dd61f6106a56255baab031acc4257bddaeddb3a1003b4ca3fd"}, @@ -3917,7 +4939,7 @@ description = "CUDNN FrontEnd python library" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "nvidia_cudnn_frontend-1.18.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:baa6fbc8e7c55f1c78c0374ed9a890e1cf81acaca0c92d6135d18a8e3c985244"}, {file = "nvidia_cudnn_frontend-1.18.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e4bcca42259e358002c8867e3624a558f66cd5dff2cc6c3aafd860ef2f41730"}, @@ -4043,7 +5065,7 @@ description = "NVIDIA CUTLASS Python DSL" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "nvidia_cutlass_dsl-4.4.1-py3-none-any.whl", hash = "sha256:7b8ffa0117be35ef6c9a88f4462ee2a794efd0f7d9f65090e10a953e434fbfce"}, ] @@ -4061,7 +5083,7 @@ description = "NVIDIA CUTLASS Python DSL" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "nvidia_cutlass_dsl_libs_base-4.4.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:f8e3ecf646074bea34b301be283578806e02b712f277717f3dd1b28671aefa95"}, {file = "nvidia_cutlass_dsl_libs_base-4.4.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:fb2dc0039061fa4e03db6ffc54eaebc7a3b590f45463be461f048b819bb99efe"}, @@ -4085,7 +5107,7 @@ description = "Python Bindings for the NVIDIA Management Library" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" files = [ {file = "nvidia_ml_py-13.590.48-py3-none-any.whl", hash = "sha256:fd43d30ee9cd0b7940f5f9f9220b68d42722975e3992b6c21d14144c48760e43"}, {file = "nvidia_ml_py-13.590.48.tar.gz", hash = "sha256:8184d1be52914ac7f0991cd1c0d946c65dc88a840c754cd12c274b77b88760dd"}, @@ -4252,7 +5274,7 @@ description = "The official Python library for the openai API" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" files = [ {file = "openai-2.24.0-py3-none-any.whl", hash = "sha256:fed30480d7d6c884303287bde864980a4b137b60553ffbcf9ab4a233b7a73d94"}, {file = "openai-2.24.0.tar.gz", hash = "sha256:1e5769f540dbd01cb33bc4716a23e67b9d695161a734aff9c5f925e2bf99a673"}, @@ -4281,7 +5303,7 @@ description = "OpenAI's response format for its open-weight model series gpt-oss optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "openai_harmony-0.0.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:029ec25ca74abe48fdb58eb9fdd2a8c1618581fc33ce8e5653f8a1ffbfbd9326"}, {file = "openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4f709815924ec325b9a890e6ab2bbb0ceec8e319a4e257328eb752cf36b2efc"}, @@ -4311,7 +5333,7 @@ description = "A stats collection and distributed tracing framework" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "opencensus-0.11.4-py2.py3-none-any.whl", hash = "sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864"}, {file = "opencensus-0.11.4.tar.gz", hash = "sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2"}, @@ -4329,7 +5351,7 @@ description = "OpenCensus Runtime Context" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "opencensus-context-0.1.3.tar.gz", hash = "sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c"}, {file = "opencensus_context-0.1.3-py2.py3-none-any.whl", hash = "sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039"}, @@ -4342,7 +5364,7 @@ description = "Wrapper package for OpenCV python bindings." optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "opencv_python_headless-4.13.0.92-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:1a7d040ac656c11b8c38677cc8cccdc149f98535089dbe5b081e80a4e5903209"}, {file = "opencv_python_headless-4.13.0.92-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:3e0a6f0a37994ec6ce5f59e936be21d5d6384a4556f2d2da9c2f9c5dc948394c"}, @@ -4364,7 +5386,7 @@ description = "OpenTelemetry Python API" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"ray\"" +markers = "(python_version >= \"3.11\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\") and (python_version == \"3.11\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\" or extra == \"client\") and (extra == \"client\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\")" files = [ {file = "opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950"}, {file = "opentelemetry_api-1.39.1.tar.gz", hash = "sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c"}, @@ -4374,6 +5396,92 @@ files = [ importlib-metadata = ">=6.0,<8.8.0" typing-extensions = ">=4.5.0" +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.39.1" +description = "OpenTelemetry Collector Exporters" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "opentelemetry_exporter_otlp-1.39.1-py3-none-any.whl", hash = "sha256:68ae69775291f04f000eb4b698ff16ff685fdebe5cb52871bc4e87938a7b00fe"}, + {file = "opentelemetry_exporter_otlp-1.39.1.tar.gz", hash = "sha256:7cf7470e9fd0060c8a38a23e4f695ac686c06a48ad97f8d4867bc9b420180b9c"}, +] + +[package.dependencies] +opentelemetry-exporter-otlp-proto-grpc = "1.39.1" +opentelemetry-exporter-otlp-proto-http = "1.39.1" + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.39.1" +description = "OpenTelemetry Protobuf encoding" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "opentelemetry_exporter_otlp_proto_common-1.39.1-py3-none-any.whl", hash = "sha256:08f8a5862d64cc3435105686d0216c1365dc5701f86844a8cd56597d0c764fde"}, + {file = "opentelemetry_exporter_otlp_proto_common-1.39.1.tar.gz", hash = "sha256:763370d4737a59741c89a67b50f9e39271639ee4afc999dadfe768541c027464"}, +] + +[package.dependencies] +opentelemetry-proto = "1.39.1" + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.39.1" +description = "OpenTelemetry Collector Protobuf over gRPC Exporter" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "opentelemetry_exporter_otlp_proto_grpc-1.39.1-py3-none-any.whl", hash = "sha256:fa1c136a05c7e9b4c09f739469cbdb927ea20b34088ab1d959a849b5cc589c18"}, + {file = "opentelemetry_exporter_otlp_proto_grpc-1.39.1.tar.gz", hash = "sha256:772eb1c9287485d625e4dbe9c879898e5253fea111d9181140f51291b5fec3ad"}, +] + +[package.dependencies] +googleapis-common-protos = ">=1.57,<2.0" +grpcio = [ + {version = ">=1.63.2,<2.0.0", markers = "python_version < \"3.13\""}, + {version = ">=1.66.2,<2.0.0", markers = "python_version >= \"3.13\""}, +] +opentelemetry-api = ">=1.15,<2.0" +opentelemetry-exporter-otlp-proto-common = "1.39.1" +opentelemetry-proto = "1.39.1" +opentelemetry-sdk = ">=1.39.1,<1.40.0" +typing-extensions = ">=4.6.0" + +[package.extras] +gcp-auth = ["opentelemetry-exporter-credential-provider-gcp (>=0.59b0)"] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.39.1" +description = "OpenTelemetry Collector Protobuf over HTTP Exporter" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "opentelemetry_exporter_otlp_proto_http-1.39.1-py3-none-any.whl", hash = "sha256:d9f5207183dd752a412c4cd564ca8875ececba13be6e9c6c370ffb752fd59985"}, + {file = "opentelemetry_exporter_otlp_proto_http-1.39.1.tar.gz", hash = "sha256:31bdab9745c709ce90a49a0624c2bd445d31a28ba34275951a6a362d16a0b9cb"}, +] + +[package.dependencies] +googleapis-common-protos = ">=1.52,<2.0" +opentelemetry-api = ">=1.15,<2.0" +opentelemetry-exporter-otlp-proto-common = "1.39.1" +opentelemetry-proto = "1.39.1" +opentelemetry-sdk = ">=1.39.1,<1.40.0" +requests = ">=2.7,<3.0" +typing-extensions = ">=4.5.0" + +[package.extras] +gcp-auth = ["opentelemetry-exporter-credential-provider-gcp (>=0.59b0)"] + [[package]] name = "opentelemetry-exporter-prometheus" version = "0.60b1" @@ -4381,7 +5489,7 @@ description = "Prometheus Metric Exporter for OpenTelemetry" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "opentelemetry_exporter_prometheus-0.60b1-py3-none-any.whl", hash = "sha256:49f59178de4f4590e3cef0b8b95cf6e071aae70e1f060566df5546fad773b8fd"}, {file = "opentelemetry_exporter_prometheus-0.60b1.tar.gz", hash = "sha256:a4011b46906323f71724649d301b4dc188aaa068852e814f4df38cc76eac616b"}, @@ -4392,6 +5500,42 @@ opentelemetry-api = ">=1.12,<2.0" opentelemetry-sdk = ">=1.39.1,<1.40.0" prometheus-client = ">=0.5.0,<1.0.0" +[[package]] +name = "opentelemetry-instrumentation" +version = "0.60b1" +description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "opentelemetry_instrumentation-0.60b1-py3-none-any.whl", hash = "sha256:04480db952b48fb1ed0073f822f0ee26012b7be7c3eac1a3793122737c78632d"}, + {file = "opentelemetry_instrumentation-0.60b1.tar.gz", hash = "sha256:57ddc7974c6eb35865af0426d1a17132b88b2ed8586897fee187fd5b8944bd6a"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.4,<2.0" +opentelemetry-semantic-conventions = "0.60b1" +packaging = ">=18.0" +wrapt = ">=1.0.0,<2.0.0" + +[[package]] +name = "opentelemetry-instrumentation-logging" +version = "0.60b1" +description = "OpenTelemetry Logging instrumentation" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "opentelemetry_instrumentation_logging-0.60b1-py3-none-any.whl", hash = "sha256:f2e18cbc7e1dd3628c80e30d243897fdc93c5b7e0c8ae60abd2b9b6a99f82343"}, + {file = "opentelemetry_instrumentation_logging-0.60b1.tar.gz", hash = "sha256:98f4b9c7aeb9314a30feee7c002c7ea9abea07c90df5f97fb058b850bc45b89a"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.12,<2.0" +opentelemetry-instrumentation = "0.60b1" + [[package]] name = "opentelemetry-proto" version = "1.39.1" @@ -4399,7 +5543,7 @@ description = "OpenTelemetry Python Proto" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" files = [ {file = "opentelemetry_proto-1.39.1-py3-none-any.whl", hash = "sha256:22cdc78efd3b3765d09e68bfbd010d4fc254c9818afd0b6b423387d9dee46007"}, {file = "opentelemetry_proto-1.39.1.tar.gz", hash = "sha256:6c8e05144fc0d3ed4d22c2289c6b126e03bcd0e6a7da0f16cedd2e1c2772e2c8"}, @@ -4415,7 +5559,7 @@ description = "OpenTelemetry Python SDK" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" files = [ {file = "opentelemetry_sdk-1.39.1-py3-none-any.whl", hash = "sha256:4d5482c478513ecb0a5d938dcc61394e647066e0cc2676bee9f3af3f3f45f01c"}, {file = "opentelemetry_sdk-1.39.1.tar.gz", hash = "sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6"}, @@ -4433,7 +5577,7 @@ description = "OpenTelemetry Semantic Conventions" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" files = [ {file = "opentelemetry_semantic_conventions-0.60b1-py3-none-any.whl", hash = "sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb"}, {file = "opentelemetry_semantic_conventions-0.60b1.tar.gz", hash = "sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953"}, @@ -4443,13 +5587,90 @@ files = [ opentelemetry-api = "1.39.1" typing-extensions = ">=4.5.0" +[[package]] +name = "orjson" +version = "3.12.0" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "(python_version >= \"3.11\" or extra == \"async-rl\") and (extra == \"async-rl\" or extra == \"client\")" +files = [ + {file = "orjson-3.12.0-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:747843254519dd43b93eee3153a19e5a509334320c4d2f823ec879232db5c796"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:7c2ad193c8004254f34b499f3bd2c80f043d10754aff2b38f93da574f4883f98"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:bc7a872f03522d90e0429e6c0c5cd23084f767bedcb4c58048eec19294613344"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18a87929f31d94a77f7dc93cf527e91f39ce7fe7813d588a4de2507efd32a387"}, + {file = "orjson-3.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9683ee9ea0659da64f36574ef675b8a86330c34c19ea75db1fb93c3ff99e0ef"}, + {file = "orjson-3.12.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:103b5db66aa53c1f9e88c2524be4f383e831ba7dfd5f9f5af6336a177c622f11"}, + {file = "orjson-3.12.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bd57d79aefa3f84eec851d6de7a366795b9345cfaf17f82b4820430a7a5fa241"}, + {file = "orjson-3.12.0-cp310-cp310-win32.whl", hash = "sha256:3dbce9b6b3074b31a5d5dd322a9c4e5b16f206091ece4194c2e36952847a105e"}, + {file = "orjson-3.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:3bb17a06f9bd15237b3216c044209fe92597379124018cfc196fbb846cde64df"}, + {file = "orjson-3.12.0-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a94f0f0c6fcbb2b5bd9734c57a489c7584a732bbdf04a39e8c83b861e9d03e92"}, + {file = "orjson-3.12.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:a696529ec96a90d9a5f9570207efe403c8b08f8e4aa2783ee3403511e2fdfa10"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:e4ac5059baab4b3acbd99485de019ff8cda0fdf34b61fa74f7197a53db78bfe8"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:8e29957429c35bbb5a185a119c523aa2428b7bbf1a293724c7b9375ed8f892a3"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dce0166feb0a737ab84f598c9a338cbc0b764a036617aa686194f53c7eba0c3e"}, + {file = "orjson-3.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9caf3d09f47c3c70c4451ada20ef9bc4a4cdffa26f49862cf0a253b329aae2d5"}, + {file = "orjson-3.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b9dca132b1fda5565088e65a6b6e742285e0aeceb6fae549fa8863e16c7d3998"}, + {file = "orjson-3.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a791f793b287bbc135b8e87c34e35c8bfc693e2a8a620fab1ae682b925f9a32e"}, + {file = "orjson-3.12.0-cp311-cp311-win32.whl", hash = "sha256:31ed278a36304390adc3eec5d7f6fd593a7c3e99e5a06cd07866396c4b1b4710"}, + {file = "orjson-3.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb2539159dfe8d371914f354360fa50e4a577cc89222a3828b9650a5e5040252"}, + {file = "orjson-3.12.0-cp311-cp311-win_arm64.whl", hash = "sha256:61318b6de893c7a9d9f3e5ecbadccbfc26a7eb417ccc7bbf0771de3b4d72f868"}, + {file = "orjson-3.12.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:aa3e43a6846e91d7bde3d5a9c66090fcd8744f569a9b6cffc5e1ca38f6a461c0"}, + {file = "orjson-3.12.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:11edb4660a6680abee9788a3a9072208a2c96538cc1322bd79542065229d8e54"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2d3a9da945a4d96ae758fdaaca56742e6b73b6fd554c5d8876f252a6dad70b83"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:92ffc09e07233a6ab6d4e067f7841edcbcc134cb4812155cf171ea5255a421d7"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf44e374aadde77b1f6109f1030be51433eb61984379852766b6f4e187db7b1e"}, + {file = "orjson-3.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1192a7021b6d071aaf909864f6e924d6a2675ca360485b972b8401749311750b"}, + {file = "orjson-3.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:53c0c474a9d9aff9aebfc0c88de1f28f843d940e6e3a80729abdf6a20274356f"}, + {file = "orjson-3.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:532ff8cd4bd59a327a953a7dcde922c7fc25b85e29721bb8633265430d3a3873"}, + {file = "orjson-3.12.0-cp312-cp312-win32.whl", hash = "sha256:a6cf4b18e7de173f209f2084ffbd736dd72389a396326ee80a7022168be232e5"}, + {file = "orjson-3.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:010811c1b69773450a01cef97727a67b223242f350b77d4ca000e59a9ef2155a"}, + {file = "orjson-3.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:ad29eece0c601737f2a60edc2752a84e7a0785df3efb62e3012834700a5afe0d"}, + {file = "orjson-3.12.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9a36ec60f1796f9a3f13e3b98390295e17a1c7c10155b448d264098bf9ee5900"}, + {file = "orjson-3.12.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ad0422b92d5195443a39f80c3bcf731cc2e00f153bd32063a47b73b057bd0f03"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5a0fdbc216388f653d3752ff310e710f59253bd4ed6a2bfb3f4f06b84714bbd8"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2eb5c56e534127b2b8fa38d2363c8b1b8190367ee0d1d16c041517d880843b94"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:784106539f4b9d4b930e0b4eb8d45168507dae001945e71b4675a367f1e5e806"}, + {file = "orjson-3.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c680706fc8396d95e7c4c1f9482563f552137aef91b57237a3ad5aaf64629df"}, + {file = "orjson-3.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:83445adc40cba26d6d621185a45128ce455b766af368cad2ab64b970603a7978"}, + {file = "orjson-3.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:644d005bc82f917337a95ce270c9f6f92f9834c2bed7b1477572f8db00784222"}, + {file = "orjson-3.12.0-cp313-cp313-win32.whl", hash = "sha256:d8e78d3d93705e3d27cc17cdb209e44d7a8ea203010cac6ce9c7ffc1ae1996f1"}, + {file = "orjson-3.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b85931be5b6763c31283805c9bdaae1ca03ad9f6f12a15f1cbf6745b907932c2"}, + {file = "orjson-3.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:6a31348d7dfa64cd9c78bd1f510ff44c48fe64d71094e6b90e364dba3b55949e"}, + {file = "orjson-3.12.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9e6fee342a48760e854d743e7a81534d8e2925a6f46e09f750cf56b50fd1de5d"}, + {file = "orjson-3.12.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:8c3bb86dd10f39b3fbf434b7d5dc7cac77d6fc8ac572ae30a10731ede2c4b647"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2bb3ce43203936072dd8b4917b01d3aecfc02329bfb42510cb7cfb24708adc9c"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:6a2a79c89984dc719817d388c8709e0efc2a2795a934eaa746b4882eb6045adc"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f06dd838d1e07d9b1de0932ec0485ec92c4d5f5d1ad4817a656268c3e88be1e1"}, + {file = "orjson-3.12.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6b11be792c3d2c6a4be2af4ebf97a68d0bf5f580aca6e86a418a354f6cc846a"}, + {file = "orjson-3.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:477ecaf6b9f88f873341b91fcc736119ca81b5e002a9f7f308ff5b4f2ce2a70e"}, + {file = "orjson-3.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f3c0683136acdc29afdf88a5bc2f7d3d0e34087788d1d63c0144b805a87a196f"}, + {file = "orjson-3.12.0-cp314-cp314-win32.whl", hash = "sha256:d39f3f5c3927e2dc0913fe5bbc1a2f6b1b9d1bba1de6358340d0ad0d0c00ca92"}, + {file = "orjson-3.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:0b1ac5bf6609b2716c7954011c5fef6254922df029f45d032ee4ebf5d363cbed"}, + {file = "orjson-3.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:50fae885cb073eac7556353ff3df93312b0d5137b0a5056b2bb63f97ed9a93c7"}, + {file = "orjson-3.12.0-cp315-cp315-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:01efac2074fffb4cb1ea3fab7861e9d0f2a26913854a972f5ac760525dbdaf6e"}, + {file = "orjson-3.12.0-cp315-cp315-macosx_15_0_arm64.whl", hash = "sha256:ed4ca42bd55955aa34deedcfdfd0e0c31abf51143aae158ae2bc3520b626e517"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_aarch64.whl", hash = "sha256:40f92192227505acca4e2533ce565f8e6b9535f7d0d09b0968452f18b7376b38"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_armv7l.whl", hash = "sha256:33efefcf5d88eaf400b47e2eba02f91f319bb9951be61ca500b7d536d3f2079d"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_i686.whl", hash = "sha256:8e386b0bc0ddd7cd2056f884b5a0af33592bd01ac66a7ca4b42a65a7e7774a13"}, + {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_x86_64.whl", hash = "sha256:58c58e1de0006ffb580368d6793c36c7b0b021db066479cf281bf5061e732328"}, + {file = "orjson-3.12.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:08231552159be266a7269555bd9f7c016aee7d9ad6dab06eb58796c5ccb7101c"}, + {file = "orjson-3.12.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a15f9a891bce5f5cc5d210e3ad8614d4d1b489a56448c099d6d2a7168b2d954a"}, + {file = "orjson-3.12.0-cp315-cp315-win32.whl", hash = "sha256:03091c8a64db4be38746597ceea68f33c238e27acd9bfe99fb59420224ae7a55"}, + {file = "orjson-3.12.0-cp315-cp315-win_amd64.whl", hash = "sha256:2b7bcefb9f40fa242fa6b06377232c048e655747790829609168c01162f60578"}, + {file = "orjson-3.12.0-cp315-cp315-win_arm64.whl", hash = "sha256:859fc4196855890150bb08e649b30d2c93b249b3e3edd0d3bb2231abf8aa8adc"}, + {file = "orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5"}, +] + [[package]] name = "oss2" version = "2.13.1" description = "Aliyun OSS (Object Storage Service) SDK" -optional = false +optional = true python-versions = "*" groups = ["main"] +markers = "extra == \"eval\"" files = [ {file = "oss2-2.13.1.tar.gz", hash = "sha256:8548ea7d43326f6fd679bc8b79b3a2dfbfe9c6a60ed57e2410818fec57023dda"}, ] @@ -4469,7 +5690,7 @@ description = "Structured Text Generation in Rust" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "outlines_core-0.2.11-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:89d79d8454b321f60047541a896d410ca9db631d241960266c4fe839cf5cd1b1"}, {file = "outlines_core-0.2.11-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:44d581893f8644da02db7be11887229a40d26077cbdd22072ad1ed1db0ad0b2d"}, @@ -4517,6 +5738,19 @@ files = [ [package.extras] test = ["asv", "coverage[toml] (>=5.1)", "diff-cover", "maturin[patchelf]", "numba", "numpy", "pre-commit", "psutil", "pydantic", "pytest", "pytest-benchmark", "pytest-cov", "pytest-mock", "scipy", "torch"] +[[package]] +name = "overrides" +version = "7.7.0" +description = "A decorator to automatically detect mismatch when overriding a method." +optional = true +python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49"}, + {file = "overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a"}, +] + [[package]] name = "packaging" version = "26.0" @@ -4529,6 +5763,103 @@ files = [ {file = "packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4"}, ] +[[package]] +name = "pandas" +version = "2.3.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version == \"3.10\"" +files = [ + {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, + {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, + {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, + {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, + {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, + {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, + {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, + {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, + {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, +] + +[package.dependencies] +numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + [[package]] name = "pandas" version = "3.0.1" @@ -4536,6 +5867,7 @@ description = "Powerful data structures for data analysis, time series, and stat optional = false python-versions = ">=3.11" groups = ["main"] +markers = "python_version >= \"3.11\"" files = [ {file = "pandas-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:de09668c1bf3b925c07e5762291602f0d789eca1b3a781f99c1c78f6cac0e7ea"}, {file = "pandas-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:24ba315ba3d6e5806063ac6eb717504e499ce30bd8c236d8693a5fd3f084c796"}, @@ -4625,7 +5957,7 @@ description = "Parse partial JSON generated by LLM" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "partial_json_parser-0.2.1.1.post7-py3-none-any.whl", hash = "sha256:145119e5eabcf80cbb13844a6b50a85c68bf99d376f8ed771e2a3c3b03e653ae"}, {file = "partial_json_parser-0.2.1.1.post7.tar.gz", hash = "sha256:86590e1ba6bcb6739a2dfc17d2323f028cb5884f4c6ce23db376999132c9a922"}, @@ -4780,12 +6112,117 @@ description = "A small Python package for determining appropriate platform-speci optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" files = [ {file = "platformdirs-4.9.2-py3-none-any.whl", hash = "sha256:9170634f126f8efdae22fb58ae8a0eaa86f38365bc57897a6c4f781d1f5875bd"}, {file = "platformdirs-4.9.2.tar.gz", hash = "sha256:9a33809944b9db043ad67ca0db94b14bf452cc6aeaac46a88ea55b26e2e9d291"}, ] +[[package]] +name = "plotext" +version = "6.1.0" +description = "plotext plots directly on terminal" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"client\"" +files = [ + {file = "plotext-6.1.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:4fec26b86ad32b3597de5861cfbf0023bb5ae02250f0ca940b12b069bccf43e6"}, + {file = "plotext-6.1.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:bfafad58a20da6c279e722481b5c3e67a9c8c54bde722d414c5ceffe71b7c652"}, + {file = "plotext-6.1.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c035440296be20e539ee70d4de07290d84bb0c9017bb3bc5867a8b3f9cead56"}, + {file = "plotext-6.1.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e59f54cedfb56129e2aec1f885869fe13903fdd4e6f18a10301e71a7d5b9851c"}, + {file = "plotext-6.1.0-py3-none-win_amd64.whl", hash = "sha256:60c85326da5454db482c9b1357c9ba385bae4a05c9dd498ff604e80458e03e3b"}, + {file = "plotext-6.1.0.tar.gz", hash = "sha256:6b729ce6376c5c1c2152fe85e0449dcc54879d0923b3c6c8d1670108245c6f84"}, +] + +[package.extras] +image = ["pillow (>=8.4)"] +video = ["ffpyplayer (>=4.3.5)", "pillow (>=8.4)", "yt-dlp (>=2024.1.1)"] + +[[package]] +name = "plotly" +version = "7.0.0" +description = "An open-source interactive data visualization library for Python" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "plotly-7.0.0-py3-none-any.whl", hash = "sha256:78cbf7bd06d1b05bb3b8ec1b709864695229b55151b6f7530fbf55517ead6fdd"}, + {file = "plotly-7.0.0.tar.gz", hash = "sha256:08b21f1244a97e7a1a699833c4bb2678475aa108b3f1989886ed0b038ebfd849"}, +] + +[package.dependencies] +narwhals = ">=1.15.1" +packaging = "*" + +[package.extras] +dev = ["anywidget", "build", "colorcet", "fiona (<=1.9.6) ; python_version <= \"3.8\"", "geopandas", "inflect", "jupyter-builder", "jupyterlab", "kaleido (>=1.3.0)", "numpy (>=1.22)", "orjson", "pandas", "pdfrw", "pillow", "polars[timezone]", "pyarrow", "pytest", "pytz", "requests", "ruff (==0.11.12)", "scikit-image", "scipy", "statsmodels", "vaex ; python_version <= \"3.9\"", "xarray"] +dev-build = ["build", "jupyter-builder", "pytest", "requests", "ruff (==0.11.12)"] +dev-codegen = ["inflect", "pytest", "requests", "ruff (==0.11.12)"] +dev-core = ["pytest", "requests", "ruff (==0.11.12)"] +dev-optional = ["anywidget", "build", "colorcet", "fiona (<=1.9.6) ; python_version <= \"3.8\"", "geopandas", "inflect", "jupyter-builder", "jupyterlab", "kaleido (>=1.3.0)", "numpy (>=1.22)", "orjson", "pandas", "pdfrw", "pillow", "polars[timezone]", "pyarrow", "pytest", "pytz", "requests", "ruff (==0.11.12)", "scikit-image", "scipy", "statsmodels", "vaex ; python_version <= \"3.9\"", "xarray"] +dev-pandas1 = ["numpy (>=1,<2)", "pandas (>=1,<2)", "setuptools (<82)"] +dev-pandas2 = ["pandas (>=2,<3)"] +dev-pandas3 = ["pandas (>=3) ; python_version >= \"3.11\""] +express = ["numpy (>=1.22)"] +kaleido = ["kaleido (>=1.3.0)"] + +[[package]] +name = "pluggy" +version = "1.6.0" +description = "plugin and hook calling mechanisms for python" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"test\"" +files = [ + {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, + {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["coverage", "pytest", "pytest-benchmark"] + +[[package]] +name = "portalocker" +version = "4.3.2" +description = "Cross-platform file locking, with Redis, PID-file and bounded-semaphore locks" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "portalocker-4.3.2-py3-none-any.whl", hash = "sha256:42f0362e7343af78b25eb095c0cde300e746a0ab6647a835cc30764918490c93"}, + {file = "portalocker-4.3.2.tar.gz", hash = "sha256:8de4a1330f55b3c9a1d50ab1b70a765f2f136d6399ae206411117ed98c3b8e2b"}, +] + +[package.extras] +docs = ["furo", "sphinx (>=7)"] +redis = ["redis (>=5.0)"] +tests = ["coverage-conditional-plugin (>=0.9)", "fakeredis (>=2.31.0)", "portalocker[redis]", "pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-rerunfailures (>=15.1)", "pytest-timeout (>=2.1.0)", "types-pywin32 (>=310.0.0.20250429)", "typing-extensions (>=4.4)"] +win32 = ["pywin32 (>=226) ; sys_platform == \"win32\""] + +[[package]] +name = "prettytable" +version = "3.18.0" +description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "prettytable-3.18.0-py3-none-any.whl", hash = "sha256:b3346e0e6f79180833aebaac088ae926340586cf6d7d991b9eb125b65f72313a"}, + {file = "prettytable-3.18.0.tar.gz", hash = "sha256:439217116152244369caf3d9f1caf2f9fe29b03bd79e88d2928c8e718c95d680"}, +] + +[package.dependencies] +wcwidth = ">=0.3.5" + +[package.extras] +tests = ["pytest (>=9)", "pytest-cov", "pytest-lazy-fixtures"] + [[package]] name = "prometheus-client" version = "0.24.1" @@ -4793,7 +6230,7 @@ description = "Python client for the Prometheus monitoring system." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "prometheus_client-0.24.1-py3-none-any.whl", hash = "sha256:150db128af71a5c2482b36e588fc8a6b95e498750da4b17065947c16070f4055"}, {file = "prometheus_client-0.24.1.tar.gz", hash = "sha256:7e0ced7fbbd40f7b84962d5d2ab6f17ef88a72504dcf7c0b40737b43b2a461f9"}, @@ -4811,7 +6248,7 @@ description = "Instrument your FastAPI app with Prometheus metrics" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "prometheus_fastapi_instrumentator-7.1.0-py3-none-any.whl", hash = "sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9"}, {file = "prometheus_fastapi_instrumentator-7.1.0.tar.gz", hash = "sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e"}, @@ -4960,7 +6397,7 @@ description = "Beautiful, Pythonic protocol buffers" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "proto_plus-1.27.1-py3-none-any.whl", hash = "sha256:e4643061f3a4d0de092d62aa4ad09fa4756b2cbb89d4627f3985018216f9fefc"}, {file = "proto_plus-1.27.1.tar.gz", hash = "sha256:912a7460446625b792f6448bade9e55cd4e41e6ac10e27009ef71a7f317fa147"}, @@ -4979,7 +6416,7 @@ description = "" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"ray\" or extra == \"vllm\" or extra == \"megatron\"" +markers = "(python_version >= \"3.11\" or extra == \"async-rl\" or extra == \"rl\" or extra == \"server\" or extra == \"megatron\") and (extra == \"async-rl\" or extra == \"rl\" or extra == \"server\" or extra == \"megatron\" or extra == \"client\")" files = [ {file = "protobuf-6.33.5-cp310-abi3-win32.whl", hash = "sha256:d71b040839446bac0f4d162e758bea99c8251161dae9d0983a3b88dee345153b"}, {file = "protobuf-6.33.5-cp310-abi3-win_amd64.whl", hash = "sha256:3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c"}, @@ -5028,6 +6465,18 @@ files = [ dev = ["abi3audit", "black", "check-manifest", "colorama ; os_name == \"nt\"", "coverage", "packaging", "psleak", "pylint", "pyperf", "pypinfo", "pyreadline3 ; os_name == \"nt\"", "pytest", "pytest-cov", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "validate-pyproject[all]", "virtualenv", "vulture", "wheel", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] test = ["psleak", "pytest", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "setuptools", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] +[[package]] +name = "pwinput" +version = "1.0.3" +description = "A cross-platform Python module that displays **** for password input. Works on Windows, unlike getpass. Formerly called stdiomask." +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "pwinput-1.0.3.tar.gz", hash = "sha256:ca1a8bd06e28872d751dbd4132d8637127c25b408ea3a349377314a5491426f3"}, +] + [[package]] name = "py-cpuinfo" version = "9.0.0" @@ -5035,12 +6484,25 @@ description = "Get CPU info with pure Python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690"}, {file = "py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5"}, ] +[[package]] +name = "py-data-juicer" +version = "1.1.0" +description = "A One-Stop Data Processing System for Large Language Models." +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"data\"" +files = [ + {file = "py_data_juicer-1.1.0-py3-none-any.whl", hash = "sha256:27faf8ab1ee4a9597c09b4f9f18286a2292b1d7fb4ffe08b94ab5a1108489221"}, + {file = "py_data_juicer-1.1.0.tar.gz", hash = "sha256:675dca2b0fde9341c676351a29aec7cfa0a756608d64160a2745fe07e0fb4bf2"}, +] + [[package]] name = "py-spy" version = "0.4.1" @@ -5048,7 +6510,7 @@ description = "" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "py_spy-0.4.1-py2.py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:809094208c6256c8f4ccadd31e9a513fe2429253f48e20066879239ba12cd8cc"}, {file = "py_spy-0.4.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:1fb8bf71ab8df95a95cc387deed6552934c50feef2cf6456bc06692a5508fd0c"}, @@ -5130,7 +6592,7 @@ description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "pyasn1-0.6.2-py3-none-any.whl", hash = "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf"}, {file = "pyasn1-0.6.2.tar.gz", hash = "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b"}, @@ -5143,7 +6605,7 @@ description = "A collection of ASN.1-based protocols modules" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, @@ -5159,7 +6621,7 @@ description = "Fast Base64 encoding/decoding" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "pybase64-1.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f63aa7f29139b8a05ce5f97cdb7fad63d29071e5bdc8a638a343311fe996112a"}, {file = "pybase64-1.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f5943ec1ae87a8b4fe310905bb57205ea4330c75e2c628433a7d9dd52295b588"}, @@ -5385,7 +6847,7 @@ description = "ISO country, subdivision, language, currency and script definitio optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "pycountry-26.2.16-py3-none-any.whl", hash = "sha256:115c4baf7cceaa30f59a4694d79483c9167dbce7a9de4d3d571c5f3ea77c305a"}, {file = "pycountry-26.2.16.tar.gz", hash = "sha256:5b6027d453fcd6060112b951dd010f01f168b51b4bf8a1f1fc8c95c8d94a0801"}, @@ -5395,10 +6857,10 @@ files = [ name = "pycparser" version = "3.0" description = "C parser in Python" -optional = true +optional = false python-versions = ">=3.10" groups = ["main"] -markers = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\" and (extra == \"vllm\" or extra == \"ray\") or extra == \"vllm\" and implementation_name == \"pypy\"" +markers = "(platform_python_implementation != \"PyPy\" or implementation_name == \"pypy\") and (extra == \"rl\" or extra == \"async-rl\" or implementation_name == \"pypy\") and implementation_name != \"PyPy\"" files = [ {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, @@ -5408,9 +6870,10 @@ files = [ name = "pycryptodome" version = "3.23.0" description = "Cryptographic library for Python" -optional = false +optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" groups = ["main"] +markers = "extra == \"eval\"" files = [ {file = "pycryptodome-3.23.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a176b79c49af27d7f6c12e4b178b0824626f40a7b9fed08f712291b6d54bf566"}, {file = "pycryptodome-3.23.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:573a0b3017e06f2cffd27d92ef22e46aa3be87a2d317a5abf7cc0e84e321bd75"}, @@ -5619,7 +7082,7 @@ description = "Extra Pydantic types." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "pydantic_extra_types-2.11.0-py3-none-any.whl", hash = "sha256:84b864d250a0fc62535b7ec591e36f2c5b4d1325fa0017eb8cda9aeb63b374a6"}, {file = "pydantic_extra_types-2.11.0.tar.gz", hash = "sha256:4e9991959d045b75feb775683437a97991d02c138e00b59176571db9ce634f0e"}, @@ -5646,7 +7109,7 @@ description = "Settings management using Pydantic" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "pydantic_settings-2.13.1-py3-none-any.whl", hash = "sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237"}, {file = "pydantic_settings-2.13.1.tar.gz", hash = "sha256:b4c11847b15237fb0171e1462bf540e294affb9b86db4d9aa5c01730bdbe4025"}, @@ -5694,14 +7157,37 @@ doc = ["ablog (>=0.11.8)", "colorama", "graphviz", "ipykernel", "ipyleaflet", "i i18n = ["Babel", "jinja2"] test = ["pytest", "pytest-cov", "pytest-regressions", "sphinx[test]"] +[[package]] +name = "pyecharts" +version = "2.1.0" +description = "Python options, make charting easier" +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "pyecharts-2.1.0-py3-none-any.whl", hash = "sha256:5cba0aa06af3d9390a1fed18aa4e76acd7bdfb323cdf131874f46eaf7e78c6c7"}, + {file = "pyecharts-2.1.0.tar.gz", hash = "sha256:077f8205390aea705b8cc0be9d0c09916a2c060e9e5c8d9c9d11b7d8f4bdc1da"}, +] + +[package.dependencies] +jinja2 = "*" +prettytable = "*" +simplejson = "*" + +[package.extras] +images = ["PIL"] +phantomjs = ["snapshot-phantomjs"] +pyppeteer = ["snapshot-pyppeteer"] +selenium = ["snapshot-selenium"] + [[package]] name = "pygments" version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." -optional = true +optional = false python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"tinker\" or extra == \"docs\"" files = [ {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, @@ -5717,7 +7203,7 @@ description = "JSON Web Token implementation in Python" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "pyjwt-2.11.0-py3-none-any.whl", hash = "sha256:94a6bde30eb5c8e04fee991062b534071fd1439ef58d2adc9ccb823e7bcd0469"}, {file = "pyjwt-2.11.0.tar.gz", hash = "sha256:35f95c1f0fbe5d5ba6e43f00271c275f7a1a4db1dab27bf708073b75318ea623"}, @@ -5732,6 +7218,237 @@ dev = ["coverage[toml] (==7.10.7)", "cryptography (>=3.4.0)", "pre-commit", "pyt docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==7.10.7)", "pytest (>=8.4.2,<9.0.0)"] +[[package]] +name = "pylatexenc" +version = "2.11" +description = "Simple LaTeX parser providing latex-to-unicode and unicode-to-latex conversion" +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "pylatexenc-2.11-py2.py3-none-any.whl", hash = "sha256:e78e7391d6c104f1ed150e21cfaa58016cdb50aa54406a2eecb793649ffdfdd0"}, + {file = "pylatexenc-2.11.tar.gz", hash = "sha256:305a072a99ce736246049c9da05841b9d718c0f7ea8888f5f596cf15cb621053"}, +] + +[[package]] +name = "pynvml" +version = "13.0.1" +description = "Python utilities for the NVIDIA Management Library" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "pynvml-13.0.1-py3-none-any.whl", hash = "sha256:e2b20e0a501eeec951e2455b7ab444759cf048e0e13a57b08049fa2775266aa8"}, + {file = "pynvml-13.0.1.tar.gz", hash = "sha256:1245991d9db786b4d2f277ce66869bd58f38ac654e38c9397d18f243c8f6e48f"}, +] + +[package.dependencies] +nvidia-ml-py = ">=12.0.0" + +[package.extras] +test = ["pytest (>=3.6)", "pytest-cov", "pytest-runner"] + +[[package]] +name = "pyqwest" +version = "0.10.0" +description = "A modern, high-performance HTTP client for Python and Rust." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.11\" and extra == \"client\"" +files = [ + {file = "pyqwest-0.10.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:aa492d5777dd145a60795ed95d9d4707a3cd1091fdcdfc93a82ac7fdc43ebacd"}, + {file = "pyqwest-0.10.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:59f3f16628e518c674102e7b5fcff2101bba6abb4f6737ec5fade9b9278e6a53"}, + {file = "pyqwest-0.10.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6e7db305a8318b1f3218053e87501f8f245ca8bd63e948e0282d04bf0883470"}, + {file = "pyqwest-0.10.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5c757cfac5f53c8671dcb4850d5fc4c4339ea3e90636331c9318f8e3ddabc06"}, + {file = "pyqwest-0.10.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:234b3f71e3f314d997c203d8cf829b7117edd041153f9c277d0060ab90134148"}, + {file = "pyqwest-0.10.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5637256a0dac0ef57e0eaa02b032014965e4a4c995e1deca1b1b97e6d1765f78"}, + {file = "pyqwest-0.10.0-cp310-abi3-win_amd64.whl", hash = "sha256:7ea761937acf3a00d1a7e70e982949d18946e5471d1419266ab3a78bbfa19759"}, + {file = "pyqwest-0.10.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a21f1f15252a8303623b4f17b9c6de595ace11b3ade07f2adb6d07121e8191aa"}, + {file = "pyqwest-0.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:eb472c6e5d6833ebfec79db310e426eb17b01ac64c0e2c251bd9192c0d2ee0c5"}, + {file = "pyqwest-0.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83578e24cccd5e0dc04d60a0af7bfb43325b5f22d03ff74ff79ed0ecf553b50d"}, + {file = "pyqwest-0.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bb511c434f79c641efb5573e5795e56dc972252f4b96e52a9636d4ece5231a4"}, + {file = "pyqwest-0.10.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1aaccd8a9db9430b2aedb5bad8ead80742cbc056b85c229516c70dc80539f906"}, + {file = "pyqwest-0.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:73d9eb438ab4a957a1ce0619d3af8c1c1126bfb9181033b123d792fcf4224531"}, + {file = "pyqwest-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:317a74d633abe3bc5bccabf479e069c515dab9e6a755274b0ccb1d8a5bbfede3"}, + {file = "pyqwest-0.10.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3978e794b9cfd8eaa500fb5d7aee63bc6172c605efa0abc1f62d85485bc049e1"}, + {file = "pyqwest-0.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:715991fd4f04862cd7a9d7452daabcdbd74dff4dff55eb20c22d60382dc2a4ed"}, + {file = "pyqwest-0.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c04798bed79c1dfa0e5b0e30fb137124311083490d44d6dfbe068d3dd254349e"}, + {file = "pyqwest-0.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35b472877e73dd63fed089c2bc8fa198407f005c8c19e0a93f025ebefde01a81"}, + {file = "pyqwest-0.10.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:564ec360b7848b35e009038ffbca00466305a9708ab21829477f64aa8cad4c64"}, + {file = "pyqwest-0.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b5c80e88a5967c1cadb3237c450f91a84a3683f8838c8dca96f09fee3612e762"}, + {file = "pyqwest-0.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:dc3d80b402fb59dbe015e25993ac8147456fb231a4c949f92a89f31315ad50f9"}, + {file = "pyqwest-0.10.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:23a28beb55fa6d975949bffae4adfb69378f3229bb5cbd71231e95bf66f5b26c"}, + {file = "pyqwest-0.10.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e4415ae40b8eedb1713dab14d7f9fecc3f79d26f3206c561087b88b99d5ce24b"}, + {file = "pyqwest-0.10.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14b875d2273212d7fa8e4b755d8d736ffd226b1c707a9c0017dfdc8393a96eca"}, + {file = "pyqwest-0.10.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5071491e416ea54e3b95bf9ffbed0bd065b093cb96e10a75c3d8f2cbe3c9823"}, + {file = "pyqwest-0.10.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b68b5e68d513a4c63a072f8f40e38015160cf90bfbf7e8ef7c3935ca87e9e022"}, + {file = "pyqwest-0.10.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c48910d27820b9c46fcd001b0fe514a3cf47d4784f59512dcdb8c91c395f82e4"}, + {file = "pyqwest-0.10.0-cp314-cp314-win_amd64.whl", hash = "sha256:d03ba2cd17948b623a6210981d342eb122546d8a8e910ec77511aff4b1acdd00"}, + {file = "pyqwest-0.10.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:07a0eb595f4096232c2d22549b6e4612c1ecada7934e46462c2c37ce14a89cfb"}, + {file = "pyqwest-0.10.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:26401baf7dafc71c8d12d2e8389519d141e6f7c14094d0dd4cf9ec1d3b5555bd"}, + {file = "pyqwest-0.10.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5e3c436e041d8873ce5bb0fdcf9f9e86f5604e8f0ef9e03149efebd8cb474f6"}, + {file = "pyqwest-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09364115761579eabfc79d1e954cdb3ded508dac1903fac7285d4c6f058c683f"}, + {file = "pyqwest-0.10.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559674a98a8b1217e1830ecd41c9905bf2b60983c6b8017063dfac199f00727c"}, + {file = "pyqwest-0.10.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f399a696392fff3db3eef0a18ef65b8a3b8396d129193487d966b8eb11006376"}, + {file = "pyqwest-0.10.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0f9163d6dd991bf1bf27308ba38ba021af660b15fffa47ebca98e41cf6f00309"}, + {file = "pyqwest-0.10.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cfcc7ba0229baa17831582befb046ace167b368140dae022d0b89b8d586ba12c"}, + {file = "pyqwest-0.10.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:eff9ccf427604d34c635954def07b6113d4754f968073eef2df40bdb80b05bf5"}, + {file = "pyqwest-0.10.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78662158093f9d5c742368f4dd9956aa595f44c6ac0860777c98853ddd5e1610"}, + {file = "pyqwest-0.10.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0879a3f0b37876372aa328b9fee956165db174c1ab72464146fe515f49399ddb"}, + {file = "pyqwest-0.10.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:850b8de6ade09a60bdb2f969871a177c2c304b594b2034ba3f5962c7bea75551"}, + {file = "pyqwest-0.10.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:399802647ea646c6ac9b5460e541b7c209b7a13563c667b2690ded2060185f2e"}, + {file = "pyqwest-0.10.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c26f3de1feb5d066d7a66802a47407a93ba043696064ad80beda4a0a4bf10056"}, + {file = "pyqwest-0.10.0.tar.gz", hash = "sha256:6c1a693be17d57d2c2eca4085e32c2809c53090c16719a907c90ebcf1f40dc01"}, +] + +[package.dependencies] +opentelemetry-api = ">=1.39.1" + +[[package]] +name = "pyroaring" +version = "1.0.3" +description = "Library for handling efficiently sorted integer sets." +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "pyroaring-1.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c10e4cfbe203a578c78808406af491e3615d5e46cf69a7709050243346cd68bc"}, + {file = "pyroaring-1.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cc329c62e504f2531c4008240f31736bcd2dee4339071f1eac0648068e6d17fa"}, + {file = "pyroaring-1.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c7fb6ddf6ef31148f0939bc5c26b681d63df301ee1e372525012dd7bfe4a30a"}, + {file = "pyroaring-1.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd18446832ea04a7d33bd6b78270b0be14eabcda5937af3428d6cb3d2bf98e54"}, + {file = "pyroaring-1.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f0cbc766df2a24e28f23d69b66bbec64e691799219fd82c2f2236f03fc88e2e"}, + {file = "pyroaring-1.0.3-cp310-cp310-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:96a51e96f8f473381615f0f852f7238ad0a47f28e4a35e9f082468c5cfe4e9c3"}, + {file = "pyroaring-1.0.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:654af38b1f1c9bdc27b4f6d331fc5d91599df96e72a6df1886f4d95eea60ab29"}, + {file = "pyroaring-1.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6721036afa31c07bdcbb4fcafa166660cf9c2eac695dcd495f8778549fa55899"}, + {file = "pyroaring-1.0.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:0caa10f20329d09233fac6550b2adce4d9f173f748a9a9a5ea3b7033827dfe2d"}, + {file = "pyroaring-1.0.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f109be8af937e85c52cb920d3fd120db52b172f59460852d2e3d2e3d13a4f52a"}, + {file = "pyroaring-1.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ddc80bfcd313c7c524a2742d263e73cae088b6a611b77dcc46fa90c306f6dace"}, + {file = "pyroaring-1.0.3-cp310-cp310-win32.whl", hash = "sha256:5a183f5ec069757fe5b60e37f7c6fa8a53178eacf0d76601b739e2890edee036"}, + {file = "pyroaring-1.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:051bd9a66ce855a1143faa2b879ea6c6ca2905209e172ce9eedf79834897c730"}, + {file = "pyroaring-1.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:3043ff5c85375310ca3cd3e01944e03026e0ec07885e52dfabcfcd9dc303867f"}, + {file = "pyroaring-1.0.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:755cdac1f9a1b7b5c621e570d4f6dbcf3b8e4a1e35a66f976104ecb35dce4ed2"}, + {file = "pyroaring-1.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ebab073db620f26f0ba11e13fa2f35e3b1298209fba47b6bc8cb6f0e2c9627f9"}, + {file = "pyroaring-1.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:684fb8dffe19bdb7f91897c65eac6eee23b1e46043c47eb24288f28a1170fe04"}, + {file = "pyroaring-1.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:678d31fc24e82945a1bfb14816c77823983382ffea76985d494782aa2f058427"}, + {file = "pyroaring-1.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d815f624e0285db3669f673d1725cb754b120ec70d0032d7c7166103a96c96d"}, + {file = "pyroaring-1.0.3-cp311-cp311-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:57fd5b80dacb8e888402b6b7508a734c6a527063e4e24e882ff2e0fd90721ada"}, + {file = "pyroaring-1.0.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab26a7a45a0bb46c00394d1a60a9f2d57c220f84586e30d59b39784b0f94aee6"}, + {file = "pyroaring-1.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9232f3f606315d59049c128154100fd05008d5c5c211e48b21848cd41ee64d26"}, + {file = "pyroaring-1.0.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f34b44b3ec3df97b978799f2901fefb2a48d367496fd1cde3cc5fe8b3bc13510"}, + {file = "pyroaring-1.0.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:25a83ec6bac3106568bd3fdd316f0fee52aa0be8c72da565ad02b10ae7905924"}, + {file = "pyroaring-1.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c17d4ec53b5b6b333d9a9515051213a691293ada785dc8c025d3641482597ed3"}, + {file = "pyroaring-1.0.3-cp311-cp311-win32.whl", hash = "sha256:d54024459ace600f1d1ffbc6dc3c60eb47cca3b678701f06148f59e10f6f8d7b"}, + {file = "pyroaring-1.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:c28750148ef579a7447a8cb60b39e5943e03f8c29bce8f2788728f6f23d1887a"}, + {file = "pyroaring-1.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:535d8deccbd8db2c6bf38629243e9646756905574a742b2a72ff51d6461d616c"}, + {file = "pyroaring-1.0.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:add3e4c78eb590a76526ecce8d1566eecdd5822e351c36b3697997f4a80ed808"}, + {file = "pyroaring-1.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ebaffe846cf4ba4f00ce6b8a9f39613f24e2d09447e77be4fa6e898bc36451b6"}, + {file = "pyroaring-1.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9459f27498f97d08031a34a5ead230b77eb0ab3cc3d85b7f54faa2fd548acd6"}, + {file = "pyroaring-1.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2b2eb8bd1c35c772994889be9f7dda09477475d7aa1e2af9ab4ef18619326f6"}, + {file = "pyroaring-1.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d31f4c1c906f1af14ce61a3959d04a14a64c594f8a768399146a45bbd341f21f"}, + {file = "pyroaring-1.0.3-cp312-cp312-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:53be988fc86698d56c11049bfe5113a2f6990adb1fa2782b29636509808b6aa7"}, + {file = "pyroaring-1.0.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7df84d223424523b19a23781f4246cc247fd6d821e1bc0853c2f25669136f7d0"}, + {file = "pyroaring-1.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:34a781f1f9766897f63ef18be129827340ae37764015b83fdcff1efb9e29136d"}, + {file = "pyroaring-1.0.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1f414343b4ed0756734328cdf2a91022fc54503769e3f8d79bd0b672ea815a16"}, + {file = "pyroaring-1.0.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d16ae185c72dc64f76335dbe53e53a892e78115adc92194957d1b7ef74d230b9"}, + {file = "pyroaring-1.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f888447bf22dde7759108bfe6dfbeb6bbb61b14948de9c4cb6843c4dd57e2215"}, + {file = "pyroaring-1.0.3-cp312-cp312-win32.whl", hash = "sha256:fbbdc44c51a0a3efd7be3dbe04466278ce098fcd101aa1905849319042159770"}, + {file = "pyroaring-1.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:3b217c4b3ad953b4c759a0d2f9bd95316f0c345b9f7adb49e6ded7a1f5106bd4"}, + {file = "pyroaring-1.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:e6bcf838564c21bab8fe6c2748b4990d4cd90612d8c470c04889def7bb5114ea"}, + {file = "pyroaring-1.0.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:20bc947054b197d1baa76cd05d70b8e04f95b82e698266e2f8f2f4b36d764477"}, + {file = "pyroaring-1.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ba5909b4c66bb85cab345e2f3a87e5ce671509c94b8c9823d8db64e107cbe854"}, + {file = "pyroaring-1.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b744746ba5da27fad760067f12633f5d384db6a1e65648d00244ceacbbd87731"}, + {file = "pyroaring-1.0.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b16c2a2791a5a09c4b59c0e1069ac1c877d0df25cae3155579c7eac8844676e"}, + {file = "pyroaring-1.0.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7f68dfcf8d01177267f4bc06c4960fe8e39577470d1b52c9af8b61a72ca8767"}, + {file = "pyroaring-1.0.3-cp313-cp313-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:dba4e4700030182a981a3c887aa73887697145fc9ffb192f908aa59b718fbbdd"}, + {file = "pyroaring-1.0.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e26dd1dc1edba02288902914bdb559e53e346e9155defa43c31fcab831b55342"}, + {file = "pyroaring-1.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6eb98d2cacfc6d51c6a69893f04075e07b3df761eac71ba162c43b9b4c4452ad"}, + {file = "pyroaring-1.0.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a967e9eddb9485cbdd95d6371e3dada67880844d836c0283d3b11efe9225d1b7"}, + {file = "pyroaring-1.0.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b12ef7f992ba7be865f91c7c098fd8ac6c413563aaa14d5b1e2bcb8cb43a4614"}, + {file = "pyroaring-1.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:82ca5be174b85c40be7b00bc6bf39b2931a1b4a465f3af17ec6b9c48e9aa6fe0"}, + {file = "pyroaring-1.0.3-cp313-cp313-win32.whl", hash = "sha256:f758c681e63ffe74b20423695e71f0410920f41b075cee679ffb5bc2bf38440b"}, + {file = "pyroaring-1.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:428c3bb384fe4c483feb5cf7aa3aef1621fb0a5c4f3d391da67b2c4a43f08a10"}, + {file = "pyroaring-1.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:9c0c856e8aa5606e8aed5f30201286e404fdc9093f81fefe82d2e79e67472bb2"}, + {file = "pyroaring-1.0.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6321a95b5b2ba69aa32e920dd1aa7f8fc4fac55b75981978aa4f2378724dee27"}, + {file = "pyroaring-1.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:054eb6ef04ff9d2ed3ddd18ae21e5e51e02d0f8cdd7e5cb948648f77ddb04ea2"}, + {file = "pyroaring-1.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4d064aea3827e63eb60294ae3e6623e29613f5c8844869646d06f3735a425dd9"}, + {file = "pyroaring-1.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c84d5b17ef628c3956d9a79c2f78c5bea7dda6f7aeb01f34671034d2650b9efb"}, + {file = "pyroaring-1.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8038f7dd25eb83c277b8e0ea14c5e61f085cc76bd0c6b9f6679f1770e33541ec"}, + {file = "pyroaring-1.0.3-cp38-cp38-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:defc508ef7acaf58d07e603c55feda6742c4034f5262cfd616f92cc3adbc2815"}, + {file = "pyroaring-1.0.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd0831326971b0ffa08ccce79abe7c2450d5d9254804d855e23a8ba31f70351a"}, + {file = "pyroaring-1.0.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:03e063329481396cbb70f1ce8b8ca0f01d74a45ee9d908b6645b0282b23832b0"}, + {file = "pyroaring-1.0.3-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:7a1b1c82d2da0bedc7c22d4047bd62544ef0e25c6be86ccf4b9d1ccc38876ee8"}, + {file = "pyroaring-1.0.3-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:dd7f9e5b7366b8f9bafca2a0fcf83fa534a00cc12d4ca01e301d8662bcdb805c"}, + {file = "pyroaring-1.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:a5a1db84e0952805223a7bf77eae58384b700a6b9affc53fb9772dddf868c712"}, + {file = "pyroaring-1.0.3-cp38-cp38-win32.whl", hash = "sha256:54cb0c2bddd330e22099773c4681aca90847265afe56a9201a92c1a758494261"}, + {file = "pyroaring-1.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:47d985293f861df1f2b03b41cef4fd3249c1c9608081750bcf3153051c2312d0"}, + {file = "pyroaring-1.0.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d46eb5db78b673d8d8ca83651a1cce1e15eec5a922f2951b1f61014463b72af5"}, + {file = "pyroaring-1.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ce202452de2b58bffa3eb02e27c681eefcfb54e27f8ef85b5c93ebaada50f3f3"}, + {file = "pyroaring-1.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:531b6ae56989b61742dde1b64fedc5537acc046cf04a333548322366c1bf3922"}, + {file = "pyroaring-1.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3035db9459bd8635a0145b4a9e3102869d621cb0b3648051115f06d31ffd1976"}, + {file = "pyroaring-1.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c8fb6b0ad0e8db1b9559b2da180b103b48adddf0e4f24404269e2a3b5db268d"}, + {file = "pyroaring-1.0.3-cp39-cp39-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8d5df95d9511bc83048da9348c7ab1c20f97ff4d95faf27ee1fdf2e8a96e200e"}, + {file = "pyroaring-1.0.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65d2d81e5aed7698fab23058db70fb2b65fad221090be037a0af498569109915"}, + {file = "pyroaring-1.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e195636034a0b62ec0e5325ed2f610f39cc8955ace3f47a5bc7f484159f02341"}, + {file = "pyroaring-1.0.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:bb7f2561e3ec26c3c869458431cbcba6b83f7e925b024460c136dbb5fadf3b31"}, + {file = "pyroaring-1.0.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8e996939de01f448eb9448d91b47ab60bff0555c2a80d5c12a8405814072cd35"}, + {file = "pyroaring-1.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c656d62d0cf96ede0edc4e7d392889238777bdf88b32afd5d51c3cab016c29a0"}, + {file = "pyroaring-1.0.3-cp39-cp39-win32.whl", hash = "sha256:a7a7d14822c64841ae64e98309697e1631ebadba55ded33daa7cd16d1b487d11"}, + {file = "pyroaring-1.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:a86b88adbe0531b75f94f87279a6d4ee68e63335e29bbdab4400a05704fc2587"}, + {file = "pyroaring-1.0.3-cp39-cp39-win_arm64.whl", hash = "sha256:1ed2e9c7af46052466b5fa0392fe540331474718d97b9756cefa23233bfdb3ea"}, + {file = "pyroaring-1.0.3.tar.gz", hash = "sha256:cd7392d1c010c9e41c11c62cd0610c8852e7e9698b1f7f6c2fcdefe50e7ef6da"}, +] + +[[package]] +name = "pytest" +version = "9.1.1" +description = "pytest: simple powerful testing with Python" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"test\"" +files = [ + {file = "pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c"}, + {file = "pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313"}, +] + +[package.dependencies] +colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1", markers = "python_version < \"3.11\""} +iniconfig = ">=1.0.1" +packaging = ">=22" +pluggy = ">=1.5,<2" +pygments = ">=2.7.2" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +description = "Pytest support for asyncio" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"test\"" +files = [ + {file = "pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1"}, + {file = "pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42"}, +] + +[package.dependencies] +backports-asyncio-runner = {version = ">=1.1,<2", markers = "python_version < \"3.11\""} +pytest = ">=8.4,<10" +typing-extensions = {version = ">=4.12", markers = "python_version < \"3.13\""} + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1)", "sphinx-tabs (>=3.5)"] +testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -5754,7 +7471,7 @@ description = "Python interpreter discovery" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "python_discovery-1.1.0-py3-none-any.whl", hash = "sha256:a162893b8809727f54594a99ad2179d2ede4bf953e12d4c7abc3cc9cdbd1437b"}, {file = "python_discovery-1.1.0.tar.gz", hash = "sha256:447941ba1aed8cc2ab7ee3cb91be5fc137c5bdbb05b7e6ea62fbdcb66e50b268"}, @@ -5775,7 +7492,7 @@ description = "Read key-value pairs from a .env file and set them as environment optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a"}, {file = "python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3"}, @@ -5791,7 +7508,7 @@ description = "JSON Log Formatter for the Python Logging Package" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "python_json_logger-4.0.0-py3-none-any.whl", hash = "sha256:af09c9daf6a813aa4cc7180395f50f2a9e5fa056034c9953aec92e381c5ba1e2"}, {file = "python_json_logger-4.0.0.tar.gz", hash = "sha256:f58e68eb46e1faed27e0f574a55a0455eecd7b8a5b88b85a784519ba3cff047f"}, @@ -5807,12 +7524,38 @@ description = "A streaming multipart parser for Python" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155"}, {file = "python_multipart-0.0.22.tar.gz", hash = "sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58"}, ] +[[package]] +name = "pytz" +version = "2026.3.post1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version == \"3.10\"" +files = [ + {file = "pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815"}, + {file = "pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d"}, +] + +[[package]] +name = "pyvers" +version = "0.2.3" +description = "A Python library for managing multiple versions of dependencies" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "pyvers-0.2.3-py3-none-any.whl", hash = "sha256:6f5b5612f2f4bd08caa49baa70fc5f875fc7da701a5385c13e244eea6b8114dd"}, + {file = "pyvers-0.2.3.tar.gz", hash = "sha256:c4b81c3a033963245e124cdecb052783c9c4cea3bb08c051833af1c44faa6283"}, +] + [[package]] name = "pywin32" version = "311" @@ -5820,7 +7563,7 @@ description = "Python for Window Extensions" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"vllm\" and sys_platform == \"win32\"" +markers = "sys_platform == \"win32\" and (extra == \"rl\" or extra == \"async-rl\")" files = [ {file = "pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3"}, {file = "pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b"}, @@ -5931,10 +7674,9 @@ files = [ name = "pyzmq" version = "27.1.0" description = "Python bindings for 0MQ" -optional = true +optional = false python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" files = [ {file = "pyzmq-27.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:508e23ec9bc44c0005c4946ea013d9317ae00ac67778bd47519fdf5a0e930ff4"}, {file = "pyzmq-27.1.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:507b6f430bdcf0ee48c0d30e734ea89ce5567fd7b8a0f0044a369c176aa44556"}, @@ -6040,7 +7782,7 @@ description = "Ray provides a simple, universal API for building distributed app optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "ray-2.54.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:a22937f09ee74a43171df338d84b45ef882c1c05748947ca9d5343a44d4b9379"}, {file = "ray-2.54.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:1e63e491155695d527513ffe9d33a6aeb3f3cdccb6309adadfd6f8dd7c0300f7"}, @@ -6060,34 +7802,71 @@ files = [ ] [package.dependencies] -aiohttp = {version = ">=3.13.3", optional = true, markers = "extra == \"serve\""} -aiohttp_cors = {version = "*", optional = true, markers = "extra == \"serve\""} +aiohttp = [ + {version = ">=3.13.3", optional = true, markers = "extra == \"serve\""}, + {version = ">=3.13.3", optional = true, markers = "extra == \"default\""}, +] +aiohttp_cors = [ + {version = "*", optional = true, markers = "extra == \"serve\""}, + {version = "*", optional = true, markers = "extra == \"default\""}, +] click = ">=7.0" -colorful = {version = "*", optional = true, markers = "extra == \"serve\""} +colorful = [ + {version = "*", optional = true, markers = "extra == \"serve\""}, + {version = "*", optional = true, markers = "extra == \"default\""}, +] cupy-cuda12x = {version = "*", optional = true, markers = "sys_platform != \"darwin\" and extra == \"cgraph\""} fastapi = {version = "*", optional = true, markers = "extra == \"serve\""} filelock = "*" -grpcio = {version = ">=1.42.0", optional = true, markers = "extra == \"serve\""} +grpcio = [ + {version = ">=1.42.0", optional = true, markers = "extra == \"serve\""}, + {version = ">=1.42.0", optional = true, markers = "extra == \"default\""}, +] jsonschema = "*" msgpack = ">=1.0.0,<2.0.0" -opencensus = {version = "*", optional = true, markers = "extra == \"serve\""} -opentelemetry-exporter-prometheus = {version = "*", optional = true, markers = "extra == \"serve\""} -opentelemetry-proto = {version = "*", optional = true, markers = "extra == \"serve\""} -opentelemetry-sdk = {version = ">=1.30.0", optional = true, markers = "extra == \"serve\""} +opencensus = [ + {version = "*", optional = true, markers = "extra == \"serve\""}, + {version = "*", optional = true, markers = "extra == \"default\""}, +] +opentelemetry-exporter-prometheus = [ + {version = "*", optional = true, markers = "extra == \"serve\""}, + {version = "*", optional = true, markers = "extra == \"default\""}, +] +opentelemetry-proto = [ + {version = "*", optional = true, markers = "extra == \"serve\""}, + {version = "*", optional = true, markers = "extra == \"default\""}, +] +opentelemetry-sdk = [ + {version = ">=1.30.0", optional = true, markers = "extra == \"serve\""}, + {version = ">=1.30.0", optional = true, markers = "extra == \"default\""}, +] packaging = ">=24.2" -prometheus_client = {version = ">=0.7.1", optional = true, markers = "extra == \"serve\""} +prometheus_client = [ + {version = ">=0.7.1", optional = true, markers = "extra == \"serve\""}, + {version = ">=0.7.1", optional = true, markers = "extra == \"default\""}, +] protobuf = ">=3.20.3" py-spy = [ {version = ">=0.2.0", optional = true, markers = "python_version < \"3.12\" and extra == \"serve\""}, + {version = ">=0.2.0", optional = true, markers = "python_version < \"3.12\" and extra == \"default\""}, {version = ">=0.4.0", optional = true, markers = "python_version >= \"3.12\" and extra == \"serve\""}, ] -pydantic = {version = "<2.0.dev0 || >=2.12.dev0,<3", optional = true, markers = "extra == \"serve\""} +pydantic = [ + {version = "<2.0.dev0 || >=2.12.dev0,<3", optional = true, markers = "extra == \"serve\""}, + {version = "<2.0.dev0 || >=2.12.dev0,<3", optional = true, markers = "extra == \"default\""}, +] pyyaml = "*" requests = "*" -smart_open = {version = "*", optional = true, markers = "extra == \"serve\""} +smart_open = [ + {version = "*", optional = true, markers = "extra == \"serve\""}, + {version = "*", optional = true, markers = "extra == \"default\""}, +] starlette = {version = "*", optional = true, markers = "extra == \"serve\""} uvicorn = {version = "*", extras = ["standard"], optional = true, markers = "extra == \"serve\""} -virtualenv = {version = ">=20.0.24,<20.21.1 || >20.21.1", optional = true, markers = "extra == \"serve\""} +virtualenv = [ + {version = ">=20.0.24,<20.21.1 || >20.21.1", optional = true, markers = "extra == \"serve\""}, + {version = ">=20.0.24,<20.21.1 || >20.21.1", optional = true, markers = "extra == \"default\""}, +] watchfiles = {version = "*", optional = true, markers = "extra == \"serve\""} [package.extras] @@ -6127,6 +7906,30 @@ commonmark = ">=0.8.1" docutils = ">=0.11" sphinx = ">=1.3.1" +[[package]] +name = "redis" +version = "8.1.0" +description = "Python client for Redis database and key-value store" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"server\"" +files = [ + {file = "redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb"}, + {file = "redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25"}, +] + +[package.dependencies] +async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} + +[package.extras] +circuit-breaker = ["pybreaker (>=1.4.0)"] +hiredis = ["hiredis (>=3.2.0)"] +jwt = ["pyjwt (>=2.13.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (>=20.0.1)", "requests (>=2.31.0)"] +otel = ["opentelemetry-api (>=1.39.1)", "opentelemetry-exporter-otlp-proto-http (>=1.39.1)", "opentelemetry-sdk (>=1.39.1)"] +xxhash = ["xxhash (>=3.6.0,<3.7.0)"] + [[package]] name = "referencing" version = "0.37.0" @@ -6134,7 +7937,7 @@ description = "JSON Referencing + Python" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231"}, {file = "referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8"}, @@ -6295,10 +8098,9 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rich" version = "14.3.3" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = true +optional = false python-versions = ">=3.8.0" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"tinker\"" files = [ {file = "rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d"}, {file = "rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b"}, @@ -6318,7 +8120,7 @@ description = "Rich toolkit for building command-line applications" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "rich_toolkit-0.19.7-py3-none-any.whl", hash = "sha256:0288e9203728c47c5a4eb60fd2f0692d9df7455a65901ab6f898437a2ba5989d"}, {file = "rich_toolkit-0.19.7.tar.gz", hash = "sha256:133c0915872da91d4c25d85342d5ec1dfacc69b63448af1a08a0d4b4f23ef46e"}, @@ -6336,7 +8138,7 @@ description = "Python Bindings for the ignore crate" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "rignore-0.7.6-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:f3c74a7e5ee77aea669c95fdb3933f2a6c7549893700082e759128a29cf67e45"}, {file = "rignore-0.7.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7202404958f5fe3474bac91f65350f0b1dde1a5e05089f2946549b7e91e79ec"}, @@ -6494,6 +8296,40 @@ files = [ {file = "rignore-0.7.6.tar.gz", hash = "sha256:00d3546cd793c30cb17921ce674d2c8f3a4b00501cb0e3dd0e82217dbeba2671"}, ] +[[package]] +name = "rouge-chinese" +version = "1.0.3" +description = "Python ROUGE Score Implementation for Chinese Language Task (official rouge score)" +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "rouge_chinese-1.0.3-py3-none-any.whl", hash = "sha256:afda22cb64f9a0f1d7a19e88d7cd9acfd2c1f5d45bf463ade7ecadf87f30624d"}, + {file = "rouge_chinese-1.0.3.tar.gz", hash = "sha256:4074de21b01a83ca6032fe6d6354c5f618a305da60c994294133be2abf030f4a"}, +] + +[package.dependencies] +six = "*" + +[[package]] +name = "rouge-score" +version = "0.1.2" +description = "Pure python implementation of ROUGE-1.5.5." +optional = true +python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, +] + +[package.dependencies] +absl-py = "*" +nltk = "*" +numpy = "*" +six = ">=1.14.0" + [[package]] name = "rpds-py" version = "0.30.0" @@ -6501,7 +8337,7 @@ description = "Python bindings to Rust's persistent data structures (rpds)" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288"}, {file = "rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00"}, @@ -6627,7 +8463,7 @@ description = "Pure-Python RSA implementation" optional = true python-versions = "<4,>=3.6" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762"}, {file = "rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75"}, @@ -6636,6 +8472,32 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "sacrebleu" +version = "2.6.0" +description = "Hassle-free computation of shareable, comparable, and reproducible BLEU, chrF, and TER scores" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "sacrebleu-2.6.0-py3-none-any.whl", hash = "sha256:3edc1531575cfe4ad04ce53491a9307e234af1c3f805a1f491cbec844229a8a8"}, + {file = "sacrebleu-2.6.0.tar.gz", hash = "sha256:91499b6cd46138d95154fff1e863c2f9be57e82f0c719d8dd718d0006cf6c566"}, +] + +[package.dependencies] +colorama = "*" +lxml = "*" +numpy = ">=1.17" +portalocker = "*" +regex = "*" +tabulate = ">=0.8.9" + +[package.extras] +dev = ["lxml-stubs", "mypy", "pytest", "setuptools", "types-tabulate", "wheel"] +ja = ["ipadic (>=1.0,<2.0)", "mecab-python3 (>=1.0.9,<2.0.0)"] +ko = ["mecab-ko (>=1.0.2,<2.0.0)", "mecab-ko-dic (>=1.0,<2.0)"] + [[package]] name = "safetensors" version = "0.7.0" @@ -6683,6 +8545,71 @@ testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2 testingfree = ["huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools-rust (>=1.5.2)"] torch = ["packaging", "safetensors[numpy]", "torch (>=1.10)"] +[[package]] +name = "scipy" +version = "1.15.3" +description = "Fundamental algorithms for scientific computing in Python" +optional = false +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version == \"3.10\"" +files = [ + {file = "scipy-1.15.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a345928c86d535060c9c2b25e71e87c39ab2f22fc96e9636bd74d1dbf9de448c"}, + {file = "scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ad3432cb0f9ed87477a8d97f03b763fd1d57709f1bbde3c9369b1dff5503b253"}, + {file = "scipy-1.15.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:aef683a9ae6eb00728a542b796f52a5477b78252edede72b8327a886ab63293f"}, + {file = "scipy-1.15.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:1c832e1bd78dea67d5c16f786681b28dd695a8cb1fb90af2e27580d3d0967e92"}, + {file = "scipy-1.15.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:263961f658ce2165bbd7b99fa5135195c3a12d9bef045345016b8b50c315cb82"}, + {file = "scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2abc762b0811e09a0d3258abee2d98e0c703eee49464ce0069590846f31d40"}, + {file = "scipy-1.15.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed7284b21a7a0c8f1b6e5977ac05396c0d008b89e05498c8b7e8f4a1423bba0e"}, + {file = "scipy-1.15.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5380741e53df2c566f4d234b100a484b420af85deb39ea35a1cc1be84ff53a5c"}, + {file = "scipy-1.15.3-cp310-cp310-win_amd64.whl", hash = "sha256:9d61e97b186a57350f6d6fd72640f9e99d5a4a2b8fbf4b9ee9a841eab327dc13"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:993439ce220d25e3696d1b23b233dd010169b62f6456488567e830654ee37a6b"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:34716e281f181a02341ddeaad584205bd2fd3c242063bd3423d61ac259ca7eba"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b0334816afb8b91dab859281b1b9786934392aa3d527cd847e41bb6f45bee65"}, + {file = "scipy-1.15.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6db907c7368e3092e24919b5e31c76998b0ce1684d51a90943cb0ed1b4ffd6c1"}, + {file = "scipy-1.15.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:721d6b4ef5dc82ca8968c25b111e307083d7ca9091bc38163fb89243e85e3889"}, + {file = "scipy-1.15.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39cb9c62e471b1bb3750066ecc3a3f3052b37751c7c3dfd0fd7e48900ed52982"}, + {file = "scipy-1.15.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:795c46999bae845966368a3c013e0e00947932d68e235702b5c3f6ea799aa8c9"}, + {file = "scipy-1.15.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:18aaacb735ab38b38db42cb01f6b92a2d0d4b6aabefeb07f02849e47f8fb3594"}, + {file = "scipy-1.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:ae48a786a28412d744c62fd7816a4118ef97e5be0bee968ce8f0a2fba7acf3bb"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6ac6310fdbfb7aa6612408bd2f07295bcbd3fda00d2d702178434751fe48e019"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:185cd3d6d05ca4b44a8f1595af87f9c372bb6acf9c808e99aa3e9aa03bd98cf6"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:05dc6abcd105e1a29f95eada46d4a3f251743cfd7d3ae8ddb4088047f24ea477"}, + {file = "scipy-1.15.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:06efcba926324df1696931a57a176c80848ccd67ce6ad020c810736bfd58eb1c"}, + {file = "scipy-1.15.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05045d8b9bfd807ee1b9f38761993297b10b245f012b11b13b91ba8945f7e45"}, + {file = "scipy-1.15.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271e3713e645149ea5ea3e97b57fdab61ce61333f97cfae392c28ba786f9bb49"}, + {file = "scipy-1.15.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cfd56fc1a8e53f6e89ba3a7a7251f7396412d655bca2aa5611c8ec9a6784a1e"}, + {file = "scipy-1.15.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ff17c0bb1cb32952c09217d8d1eed9b53d1463e5f1dd6052c7857f83127d539"}, + {file = "scipy-1.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:52092bc0472cfd17df49ff17e70624345efece4e1a12b23783a1ac59a1b728ed"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c620736bcc334782e24d173c0fdbb7590a0a436d2fdf39310a8902505008759"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:7e11270a000969409d37ed399585ee530b9ef6aa99d50c019de4cb01e8e54e62"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8c9ed3ba2c8a2ce098163a9bdb26f891746d02136995df25227a20e71c396ebb"}, + {file = "scipy-1.15.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0bdd905264c0c9cfa74a4772cdb2070171790381a5c4d312c973382fc6eaf730"}, + {file = "scipy-1.15.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79167bba085c31f38603e11a267d862957cbb3ce018d8b38f79ac043bc92d825"}, + {file = "scipy-1.15.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9deabd6d547aee2c9a81dee6cc96c6d7e9a9b1953f74850c179f91fdc729cb7"}, + {file = "scipy-1.15.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dde4fc32993071ac0c7dd2d82569e544f0bdaff66269cb475e0f369adad13f11"}, + {file = "scipy-1.15.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f77f853d584e72e874d87357ad70f44b437331507d1c311457bed8ed2b956126"}, + {file = "scipy-1.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:b90ab29d0c37ec9bf55424c064312930ca5f4bde15ee8619ee44e69319aab163"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3ac07623267feb3ae308487c260ac684b32ea35fd81e12845039952f558047b8"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6487aa99c2a3d509a5227d9a5e889ff05830a06b2ce08ec30df6d79db5fcd5c5"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:50f9e62461c95d933d5c5ef4a1f2ebf9a2b4e83b0db374cb3f1de104d935922e"}, + {file = "scipy-1.15.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:14ed70039d182f411ffc74789a16df3835e05dc469b898233a245cdfd7f162cb"}, + {file = "scipy-1.15.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a769105537aa07a69468a0eefcd121be52006db61cdd8cac8a0e68980bbb723"}, + {file = "scipy-1.15.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db984639887e3dffb3928d118145ffe40eff2fa40cb241a306ec57c219ebbbb"}, + {file = "scipy-1.15.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:40e54d5c7e7ebf1aa596c374c49fa3135f04648a0caabcb66c52884b943f02b4"}, + {file = "scipy-1.15.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5e721fed53187e71d0ccf382b6bf977644c533e506c4d33c3fb24de89f5c3ed5"}, + {file = "scipy-1.15.3-cp313-cp313t-win_amd64.whl", hash = "sha256:76ad1fb5f8752eabf0fa02e4cc0336b4e8f021e2d5f061ed37d6d264db35e3ca"}, + {file = "scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf"}, +] + +[package.dependencies] +numpy = ">=1.23.5,<2.5" + +[package.extras] +dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] +doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.19.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.0.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)"] +test = ["Cython", "array-api-strict (>=2.0,<2.1.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja ; sys_platform != \"emscripten\"", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] + [[package]] name = "scipy" version = "1.17.1" @@ -6690,6 +8617,7 @@ description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = ">=3.11" groups = ["main"] +markers = "python_version >= \"3.11\"" files = [ {file = "scipy-1.17.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec"}, {file = "scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696"}, @@ -6769,7 +8697,7 @@ description = "Unsupervised text tokenizer and detokenizer." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "sentencepiece-0.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e10fa50bdbaa5e2445dbd387979980d391760faf0ec99a09bd7780ff37eaec44"}, {file = "sentencepiece-0.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f27ae6deea72efdb6f361750c92f6c21fd0ad087445082770cc34015213c526"}, @@ -6849,7 +8777,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "sentry_sdk-2.53.0-py2.py3-none-any.whl", hash = "sha256:46e1ed8d84355ae54406c924f6b290c3d61f4048625989a723fd622aab838899"}, {file = "sentry_sdk-2.53.0.tar.gz", hash = "sha256:6520ef2c4acd823f28efc55e43eb6ce2e6d9f954a95a3aa96b6fd14871e92b77"}, @@ -6913,7 +8841,7 @@ description = "A Python module to customize the process title" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "setproctitle-1.3.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cf555b6299f10a6eb44e4f96d2f5a3884c70ce25dc5c8796aaa2f7b40e72cb1b"}, {file = "setproctitle-1.3.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:690b4776f9c15aaf1023bb07d7c5b797681a17af98a4a69e76a1d504e41108b7"}, @@ -7024,7 +8952,7 @@ description = "Easily download, build, install, upgrade, and uninstall Python pa optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_version == \"3.12\"" +markers = "python_full_version == \"3.13.0\" or python_version == \"3.12\"" files = [ {file = "setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173"}, {file = "setuptools-80.10.2.tar.gz", hash = "sha256:8b0e9d10c784bf7d262c4e5ec5d4ec94127ce206e8738f29a437945fbc219b70"}, @@ -7046,7 +8974,6 @@ description = "Easily download, build, install, upgrade, and uninstall Python pa optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_version == \"3.11\"" files = [ {file = "setuptools-82.0.0-py3-none-any.whl", hash = "sha256:70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0"}, {file = "setuptools-82.0.0.tar.gz", hash = "sha256:22e0a2d69474c6ae4feb01951cb69d515ed23728cf96d05513d36e42b62b37cb"}, @@ -7065,10 +8992,9 @@ type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.deve name = "shellingham" version = "1.5.4" description = "Tool to Detect Surrounding Shell" -optional = true +optional = false python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"vllm\"" files = [ {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, @@ -7213,7 +9139,7 @@ description = "Utils for streaming large files (S3, HDFS, GCS, SFTP, Azure Blob optional = true python-versions = "<4.0,>=3.10" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "smart_open-7.5.1-py3-none-any.whl", hash = "sha256:3e07cbbd9c8a908bcb8e25d48becf1a5cbb4886fa975e9f34c672ed171df2318"}, {file = "smart_open-7.5.1.tar.gz", hash = "sha256:3f08e16827c4733699e6b2cc40328a3568f900cb12ad9a3ad233ba6c872d9fe7"}, @@ -7240,7 +9166,7 @@ description = "Sniff out which async library your code is running under" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"tinker\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" files = [ {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, @@ -7549,7 +9475,7 @@ description = "SSE plugin for Starlette" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "sse_starlette-3.3.2-py3-none-any.whl", hash = "sha256:5c3ea3dad425c601236726af2f27689b74494643f57017cafcb6f8c9acfbb862"}, {file = "sse_starlette-3.3.2.tar.gz", hash = "sha256:678fca55a1945c734d8472a6cad186a55ab02840b4f6786f5ee8770970579dcd"}, @@ -7591,7 +9517,7 @@ description = "A system for controlling process state under UNIX" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "supervisor-4.3.0-py2.py3-none-any.whl", hash = "sha256:0bcb763fddafba410f35cbde226aa7f8514b9fb82eb05a0c85f6588d1c13f8db"}, {file = "supervisor-4.3.0.tar.gz", hash = "sha256:4a2bf149adf42997e1bb44b70c43b613275ec9852c3edacca86a9166b27e945e"}, @@ -7600,6 +9526,44 @@ files = [ [package.extras] test = ["pytest", "pytest-cov"] +[[package]] +name = "swanlab" +version = "0.10.0" +description = "Python library for streamlined tracking and management of AI training processes." +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "swanlab-0.10.0-py3-none-any.whl", hash = "sha256:5467143d68925ee0968e3409174fbf25958a7736b0c6f347c90aabaf257f9057"}, + {file = "swanlab-0.10.0.tar.gz", hash = "sha256:746ca2798a4ca40a5d0cb2535993b0f4c88348b7a182da813a4b45feda24eef9"}, +] + +[package.dependencies] +click = "*" +nvidia-ml-py = "*" +orjson = {version = "*", markers = "python_version > \"3.9\""} +platformdirs = ">=4.2.0" +protobuf = {version = ">=6.31.1,<7", markers = "python_version > \"3.9\" or sys_platform != \"linux\""} +psutil = ">=5.0.0" +pwinput = ">=1.0.3" +pydantic = ">=2.0,<3.0" +pydantic-settings = ">=2.8.1" +pyecharts = ">=2.0.0" +pyroaring = {version = ">=1.0.0,<1.0.4", markers = "python_version < \"3.14\""} +pyyaml = "*" +requests = ">=2.28.0" +rich = ">=13.6.0" +setuptools = "*" +urllib3 = ">=1.26.0" +watchdog = "*" +wrapt = ">=1.17.0" + +[package.extras] +dashboard = ["swanboard (==0.1.10b2)"] +media = ["imageio[ffmpeg]", "matplotlib", "moviepy", "numpy", "pillow", "rdkit (<=2023.9.6) ; python_version == \"3.9\"", "rdkit (>=2025.9.6) ; python_version > \"3.9\"", "soundfile"] +s3 = ["boto3 (>=1.35.49)"] + [[package]] name = "sympy" version = "1.14.0" @@ -7625,7 +9589,7 @@ description = "Pretty-print tabular data" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, @@ -7634,6 +9598,85 @@ files = [ [package.extras] widechars = ["wcwidth"] +[[package]] +name = "tensordict" +version = "0.14.2" +description = "TensorDict is a pytorch dedicated tensor container." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "tensordict-0.14.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:9e4650b76d4e0672cf9c0320225b21434a589f4629d5836114019335875ffee9"}, + {file = "tensordict-0.14.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b9963fdd3bdcfea7919eea15208d5730c7862bcb028efd26fb46b6072a2abc79"}, + {file = "tensordict-0.14.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:404f47a5d024e1215792a920d7a742402981e907bc8a0c86f374358e719fefb3"}, + {file = "tensordict-0.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:45c851e2b884c2e63180d75a82599ccb343b816a4ab3da4b28290695726184df"}, + {file = "tensordict-0.14.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e2ac08238d0404a7f17f938331918c9bea026123bc7df4c07f2ce52b3068953f"}, + {file = "tensordict-0.14.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:001c62b1b9b7305077f29ab4e1a06693f1a7add7bfb2cd42d03b2172f185d425"}, + {file = "tensordict-0.14.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:002ce1387d1b6bb2ee29affbb68aaae4c79fa8773137bef88534fc2fadefca2f"}, + {file = "tensordict-0.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:0d1f8f949db128b896b9cb022de532d3d87902811d2751c11f52270a27eabd9a"}, + {file = "tensordict-0.14.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:0e8b65def0da306741d262976e7613afb2079e8b521b6be910edd8080a33c265"}, + {file = "tensordict-0.14.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1553be455faf595bdc0e2b18649ca8791728ea13359805c22fe8c2aff6bd8fd0"}, + {file = "tensordict-0.14.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0a748755bfb35db40123bcda67bfebac0138283d25deda76fe30be58ce10c247"}, + {file = "tensordict-0.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:5e676f6048c4e3ad53de379469270545526d6a19badbc48bd5870b68c64e84f4"}, + {file = "tensordict-0.14.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8c5e4aa2a4a8b4b1370828186836fad9605f2091a3fa3e009b15423d787aaed3"}, + {file = "tensordict-0.14.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7d79bf7bcbdf6b0686ba756debffe9b73de9686e9fa5923747538408f130df9d"}, + {file = "tensordict-0.14.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:75a2a55f4633fbd3b82d158b2442e561bd595d48138ca473a2008e4e8689e640"}, + {file = "tensordict-0.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:99a36acdc59eb162d3de214be464be2c85998f004b83ddb425aa23f74b92b6f6"}, + {file = "tensordict-0.14.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:10ae025f99e23dcec102652a9859c609122abac053fc3cd09ad999959d9069f2"}, + {file = "tensordict-0.14.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:4ec2d0d1ca4a61196deadb7a88584d9df47dc720ba45d00af15783396961295f"}, + {file = "tensordict-0.14.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:6d710cbc3091be3fee8d009492fb372df4c732e064495cf0ed5cdfacb5e168bd"}, + {file = "tensordict-0.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:31512077ade1931b47b90be02a96ebfea23db28ab195c158bea01e11a0c71638"}, + {file = "tensordict-0.14.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:ad9c5a260b031145cd8a10a2fcdc5aa6c817c0e92cb80ec40bb140a0368b868a"}, + {file = "tensordict-0.14.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:595cdb56991ea6a7880af363f349299bb432041aa117fd8cdde76f0976f335a3"}, + {file = "tensordict-0.14.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:0482dd97ba526d4918f8920aaa3936f6fe32f2ed4682a714e779582eacba4c2c"}, + {file = "tensordict-0.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:68cba296f45c4eb98587644ce07902fbe75e064a2b738b8c5b2b590b8ea8f85b"}, +] + +[package.dependencies] +cloudpickle = "*" +importlib_metadata = "*" +numpy = "*" +orjson = {version = "*", markers = "python_version < \"3.13\""} +packaging = "*" +pyvers = ">=0.2.0,<0.3.0" +torch = "*" + +[package.extras] +dev = ["ninja", "pybind11 (>=2.13)"] +h5 = ["h5py (>=3.8)"] +onnx = ["onnx", "onnxruntime", "onnxscript"] +pandas = ["pandas (>=1.5)"] +parquet = ["pyarrow (>=10.0)"] +tabular = ["pandas (>=1.5)", "pyarrow (>=10.0)"] +tests = ["pytest", "pytest-benchmark", "pytest-instafail", "pytest-rerunfailures", "pyyaml"] +typecheck = ["mypy (>=1.0.0)"] +zarr = ["zarr (>=3.0) ; python_version >= \"3.11\""] + +[[package]] +name = "textual" +version = "8.2.8" +description = "Modern Text User Interface framework" +optional = true +python-versions = "<4.0,>=3.9" +groups = ["main"] +markers = "extra == \"client\"" +files = [ + {file = "textual-8.2.8-py3-none-any.whl", hash = "sha256:267375fd402dc8d981457212efa71f0e3365fd17bba144ba9bb3ed7563cb374a"}, + {file = "textual-8.2.8.tar.gz", hash = "sha256:3f106a9fbc73e39dd266c9712432087de78a6d644084c7c241d6a25c3169115b"}, +] + +[package.dependencies] +markdown-it-py = {version = ">=2.1.0", extras = ["linkify"]} +mdit-py-plugins = "*" +platformdirs = ">=3.6.0,<5" +pygments = ">=2.19.2,<3.0.0" +rich = ">=14.2.0" +typing-extensions = ">=4.4.0,<5.0.0" + +[package.extras] +syntax = ["tree-sitter (>=0.25.0) ; python_version >= \"3.10\"", "tree-sitter-bash (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-css (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-go (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-html (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-java (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-javascript (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-json (>=0.24.0) ; python_version >= \"3.10\"", "tree-sitter-markdown (>=0.3.0) ; python_version >= \"3.10\"", "tree-sitter-python (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-regex (>=0.24.0) ; python_version >= \"3.10\"", "tree-sitter-rust (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-sql (>=0.3.11) ; python_version >= \"3.10\"", "tree-sitter-toml (>=0.6.0) ; python_version >= \"3.10\"", "tree-sitter-xml (>=0.7.0) ; python_version >= \"3.10\"", "tree-sitter-yaml (>=0.6.0) ; python_version >= \"3.10\""] + [[package]] name = "tiktoken" version = "0.12.0" @@ -7641,7 +9684,7 @@ description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "tiktoken-0.12.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3de02f5a491cfd179aec916eddb70331814bd6bf764075d39e21d5862e533970"}, {file = "tiktoken-0.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b6cfb6d9b7b54d20af21a912bfe63a2727d9cfa8fbda642fd8322c70340aad16"}, @@ -7711,15 +9754,15 @@ blobfile = ["blobfile (>=2)"] [[package]] name = "tinker" -version = "0.14.0" +version = "0.29.0" description = "The official Python SDK for the tinker API" optional = true python-versions = ">=3.11" groups = ["main"] -markers = "extra == \"tinker\"" +markers = "python_version >= \"3.11\" and extra == \"client\"" files = [ - {file = "tinker-0.14.0-py3-none-any.whl", hash = "sha256:b8e4dd2c77858e80177cb0b970c5d64a7dae085b67a711cc7e1d95352534c563"}, - {file = "tinker-0.14.0.tar.gz", hash = "sha256:554b30c43d9cf83b41eb07775a2a9f65eb2e77bed2cd79792bd8cd0d7693e717"}, + {file = "tinker-0.29.0-py3-none-any.whl", hash = "sha256:d3a9edd08718edaa8d8f72fe4ffa2a7f1ae8da2967f35e1ab206988b86a1b1fb"}, + {file = "tinker-0.29.0.tar.gz", hash = "sha256:615ed096ff647addafd1637096e8c688ce1581b8132bb937426492d2c5c0d619"}, ] [package.dependencies] @@ -7728,11 +9771,15 @@ click = ">=8.0.0" distro = ">=1.7.0,<2" httpx = {version = ">=0.23.0,<1", extras = ["http2"]} numpy = "*" +orjson = ">=3.10.0" +protobuf = ">=4.21" pydantic = ">=1.9.0,<3" +pyqwest = ">=0.4.1" rich = ">=13.0.0" sniffio = "*" transformers = "*" typing-extensions = ">=4.10,<5" +zstandard = ">=0.24.0" [package.extras] aiohttp = ["aiohttp", "httpx-aiohttp (>=0.1.8)"] @@ -7780,6 +9827,64 @@ dev = ["tokenizers[testing]"] docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] testing = ["datasets", "numpy", "pytest", "pytest-asyncio", "requests", "ruff", "ty"] +[[package]] +name = "tomli" +version = "2.4.1" +description = "A lil' TOML parser" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.10\" and (extra == \"rl\" or extra == \"async-rl\" or extra == \"test\")" +files = [ + {file = "tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30"}, + {file = "tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a"}, + {file = "tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076"}, + {file = "tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9"}, + {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c"}, + {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc"}, + {file = "tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049"}, + {file = "tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e"}, + {file = "tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece"}, + {file = "tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a"}, + {file = "tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085"}, + {file = "tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9"}, + {file = "tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5"}, + {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585"}, + {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1"}, + {file = "tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917"}, + {file = "tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9"}, + {file = "tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257"}, + {file = "tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54"}, + {file = "tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a"}, + {file = "tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897"}, + {file = "tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f"}, + {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d"}, + {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5"}, + {file = "tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd"}, + {file = "tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36"}, + {file = "tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd"}, + {file = "tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf"}, + {file = "tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac"}, + {file = "tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662"}, + {file = "tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853"}, + {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15"}, + {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba"}, + {file = "tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6"}, + {file = "tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7"}, + {file = "tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232"}, + {file = "tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4"}, + {file = "tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c"}, + {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d"}, + {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41"}, + {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c"}, + {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f"}, + {file = "tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8"}, + {file = "tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26"}, + {file = "tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396"}, + {file = "tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe"}, + {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, +] + [[package]] name = "torch" version = "2.9.1" @@ -7855,7 +9960,7 @@ description = "An audio package for PyTorch" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "torchaudio-2.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd13541197e035338bd43225b2067532056486d357c661e12d49ace4fc37f8bb"}, {file = "torchaudio-2.9.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:31ec46b718b7caa0182221bfb42e2ad223947b752a996dcdc0388c34a678c966"}, @@ -7897,7 +10002,7 @@ description = "image and video datasets and models for torch deep learning" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"transformers\" or extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "torchvision-0.24.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ded5e625788572e4e1c4d155d1bbc48805c113794100d70e19c76e39e4d53465"}, {file = "torchvision-0.24.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:54ed17c3d30e718e08d8da3fd5b30ea44b0311317e55647cb97077a29ecbc25b"}, @@ -7960,6 +10065,35 @@ notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] +[[package]] +name = "transferqueue" +version = "0.1.10" +description = "TransferQueue: An Asynchronous Streaming Data Management Module" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "transferqueue-0.1.10-py3-none-any.whl", hash = "sha256:a8efceb971fe57d52ac7ddd86d9a6ef63f1824819dddc2ea980a96b73ff2f8ea"}, +] + +[package.dependencies] +hydra-core = "*" +msgspec = "*" +numpy = "*" +omegaconf = "*" +prometheus_client = ">=0.20.0" +psutil = "*" +pyzmq = "*" +ray = {version = "*", extras = ["default"]} +tensordict = ">=0.10.0" + +[package.extras] +build = ["build"] +mooncake = ["cuda-python", "mooncake-transfer-engine (>=0.3.10.post2)"] +test = ["flake8", "pytest (>=7.0.0)", "pytest-asyncio (>=0.20.0)", "pytest-mock"] +yuanrong = ["openyuanrong-datasystem"] + [[package]] name = "transformer-engine" version = "2.12.0" @@ -8134,10 +10268,9 @@ tutorials = ["matplotlib", "pandas", "tabulate"] name = "typer" version = "0.24.1" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -optional = true +optional = false python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\"" files = [ {file = "typer-0.24.1-py3-none-any.whl", hash = "sha256:112c1f0ce578bfb4cab9ffdabc68f031416ebcc216536611ba21f04e9aa84c9e"}, {file = "typer-0.24.1.tar.gz", hash = "sha256:e39b4732d65fbdcde189ae76cf7cd48aeae72919dea1fdfc16593be016256b45"}, @@ -8183,7 +10316,7 @@ description = "Provider of IANA time zone data" optional = false python-versions = ">=2" groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\"" +markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version == \"3.10\"" files = [ {file = "tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1"}, {file = "tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7"}, @@ -8214,7 +10347,7 @@ description = "The lightning-fast ASGI server." optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "uvicorn-0.41.0-py3-none-any.whl", hash = "sha256:29e35b1d2c36a04b9e180d4007ede3bcb32a85fbdfd6c6aeb3f26839de088187"}, {file = "uvicorn-0.41.0.tar.gz", hash = "sha256:09d11cf7008da33113824ee5a1c6422d89fbc2ff476540d69a34c87fab8b571a"}, @@ -8227,6 +10360,7 @@ h11 = ">=0.8" httptools = {version = ">=0.6.3", optional = true, markers = "extra == \"standard\""} python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} uvloop = {version = ">=0.15.1", optional = true, markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and extra == \"standard\""} watchfiles = {version = ">=0.20", optional = true, markers = "extra == \"standard\""} websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} @@ -8241,7 +10375,7 @@ description = "Fast implementation of asyncio event loop on top of libuv" optional = true python-versions = ">=3.8.1" groups = ["main"] -markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and (extra == \"vllm\" or extra == \"ray\")" +markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and (extra == \"rl\" or extra == \"async-rl\")" files = [ {file = "uvloop-0.22.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef6f0d4cc8a9fa1f6a910230cd53545d9a14479311e87e3cb225495952eb672c"}, {file = "uvloop-0.22.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7cd375a12b71d33d46af85a3343b35d98e8116134ba404bd657b3b1d15988792"}, @@ -8306,7 +10440,7 @@ description = "Virtual Python Environment builder" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "virtualenv-21.1.0-py3-none-any.whl", hash = "sha256:164f5e14c5587d170cf98e60378eb91ea35bf037be313811905d3a24ea33cc07"}, {file = "virtualenv-21.1.0.tar.gz", hash = "sha256:1990a0188c8f16b6b9cf65c9183049007375b26aad415514d377ccacf1e4fb44"}, @@ -8317,6 +10451,7 @@ distlib = ">=0.3.7,<1" filelock = {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""} platformdirs = ">=3.9.1,<5" python-discovery = ">=1" +typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\""} [[package]] name = "vllm" @@ -8325,7 +10460,7 @@ description = "A high-throughput and memory-efficient inference and serving engi optional = true python-versions = "<3.14,>=3.10" groups = ["main"] -markers = "extra == \"vllm\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "vllm-0.16.0-cp38-abi3-manylinux_2_31_aarch64.whl", hash = "sha256:dfaa14846608fd229dda9d372e2ad3f13854fd09147c2ba36b40579cf3c03804"}, {file = "vllm-0.16.0-cp38-abi3-manylinux_2_31_x86_64.whl", hash = "sha256:f066b2a2f8597a4a3ada8fbbfd122b59086864b2260ca42dc81bf9fb57af0c42"}, @@ -8404,6 +10539,50 @@ petit-kernel = ["petit-kernel"] runai = ["runai-model-streamer[gcs,s3] (>=0.15.3)"] tensorizer = ["tensorizer (==2.10.1)"] +[[package]] +name = "watchdog" +version = "6.0.0" +description = "Filesystem events monitoring" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26"}, + {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112"}, + {file = "watchdog-6.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c897ac1b55c5a1461e16dae288d22bb2e412ba9807df8397a635d88f671d36c3"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2"}, + {file = "watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860"}, + {file = "watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134"}, + {file = "watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b"}, + {file = "watchdog-6.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e6f0e77c9417e7cd62af82529b10563db3423625c5fce018430b249bf977f9e8"}, + {file = "watchdog-6.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:90c8e78f3b94014f7aaae121e6b909674df5b46ec24d6bebc45c44c56729af2a"}, + {file = "watchdog-6.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e7631a77ffb1f7d2eefa4445ebbee491c720a5661ddf6df3498ebecae5ed375c"}, + {file = "watchdog-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c7ac31a19f4545dd92fc25d200694098f42c9a8e391bc00bdd362c5736dbf881"}, + {file = "watchdog-6.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9513f27a1a582d9808cf21a07dae516f0fab1cf2d7683a742c498b93eedabb11"}, + {file = "watchdog-6.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7a0e56874cfbc4b9b05c60c8a1926fedf56324bb08cfbc188969777940aef3aa"}, + {file = "watchdog-6.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:e6439e374fc012255b4ec786ae3c4bc838cd7309a540e5fe0952d03687d8804e"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c"}, + {file = "watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2"}, + {file = "watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a"}, + {file = "watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680"}, + {file = "watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f"}, + {file = "watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282"}, +] + +[package.extras] +watchmedo = ["PyYAML (>=3.10)"] + [[package]] name = "watchfiles" version = "1.1.1" @@ -8411,7 +10590,7 @@ description = "Simple, modern and high performance file watching and code reload optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "watchfiles-1.1.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:eef58232d32daf2ac67f42dea51a2c80f0d03379075d44a587051e63cc2e368c"}, {file = "watchfiles-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:03fa0f5237118a0c5e496185cafa92878568b652a2e9a9382a5151b1a0380a43"}, @@ -8527,6 +10706,19 @@ files = [ [package.dependencies] anyio = ">=3.0.0" +[[package]] +name = "wcwidth" +version = "0.8.3" +description = "Measures the displayed width of unicode strings in a terminal" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"async-rl\"" +files = [ + {file = "wcwidth-0.8.3-py3-none-any.whl", hash = "sha256:d5b73dba6158a595ec9370350e7f2637bcac8d6c5e4fde34f30fcffb6103a5e4"}, + {file = "wcwidth-0.8.3.tar.gz", hash = "sha256:d128512515fbf4612e0ff21fd6380399210318b7b54a9af59dff8454cf9730eb"}, +] + [[package]] name = "websockets" version = "16.0" @@ -8534,7 +10726,7 @@ description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"vllm\" or extra == \"ray\"" +markers = "extra == \"rl\" or extra == \"async-rl\"" files = [ {file = "websockets-16.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a"}, {file = "websockets-16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0"}, @@ -8606,7 +10798,7 @@ description = "A small Python utility to set file creation time on Windows" optional = true python-versions = ">=3.5" groups = ["main"] -markers = "extra == \"vllm\" and sys_platform == \"win32\"" +markers = "sys_platform == \"win32\" and (extra == \"rl\" or extra == \"async-rl\")" files = [ {file = "win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390"}, {file = "win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0"}, @@ -8615,93 +10807,109 @@ files = [ [package.extras] dev = ["black (>=19.3b0) ; python_version >= \"3.6\"", "pytest (>=4.6.2)"] +[[package]] +name = "word2number" +version = "1.1" +description = "Convert number words eg. three hundred and forty two to numbers (342)." +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "word2number-1.1.zip", hash = "sha256:70e27a5d387f67b04c71fbb7621c05930b19bfd26efd6851e6e0f9969dcde7d0"}, +] + [[package]] name = "wrapt" -version = "2.1.1" +version = "1.17.3" description = "Module for decorators, wrappers and monkey patching." optional = true -python-versions = ">=3.9" +python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"ray\"" -files = [ - {file = "wrapt-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7e927375e43fd5a985b27a8992327c22541b6dede1362fc79df337d26e23604f"}, - {file = "wrapt-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c99544b6a7d40ca22195563b6d8bc3986ee8bb82f272f31f0670fe9440c869"}, - {file = "wrapt-2.1.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b2be3fa5f4efaf16ee7c77d0556abca35f5a18ad4ac06f0ef3904c3399010ce9"}, - {file = "wrapt-2.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67c90c1ae6489a6cb1a82058902caa8006706f7b4e8ff766f943e9d2c8e608d0"}, - {file = "wrapt-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:05c0db35ccffd7480143e62df1e829d101c7b86944ae3be7e4869a7efa621f53"}, - {file = "wrapt-2.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0c2ec9f616755b2e1e0bf4d0961f59bb5c2e7a77407e7e2c38ef4f7d2fdde12c"}, - {file = "wrapt-2.1.1-cp310-cp310-win32.whl", hash = "sha256:203ba6b3f89e410e27dbd30ff7dccaf54dcf30fda0b22aa1b82d560c7f9fe9a1"}, - {file = "wrapt-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:6f9426d9cfc2f8732922fc96198052e55c09bb9db3ddaa4323a18e055807410e"}, - {file = "wrapt-2.1.1-cp310-cp310-win_arm64.whl", hash = "sha256:69c26f51b67076b40714cff81bdd5826c0b10c077fb6b0678393a6a2f952a5fc"}, - {file = "wrapt-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c366434a7fb914c7a5de508ed735ef9c133367114e1a7cb91dfb5cd806a1549"}, - {file = "wrapt-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d6a2068bd2e1e19e5a317c8c0b288267eec4e7347c36bc68a6e378a39f19ee7"}, - {file = "wrapt-2.1.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:891ab4713419217b2aed7dd106c9200f64e6a82226775a0d2ebd6bef2ebd1747"}, - {file = "wrapt-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8ef36a0df38d2dc9d907f6617f89e113c5892e0a35f58f45f75901af0ce7d81"}, - {file = "wrapt-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:76e9af3ebd86f19973143d4d592cbf3e970cf3f66ddee30b16278c26ae34b8ab"}, - {file = "wrapt-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ff562067485ebdeaef2fa3fe9b1876bc4e7b73762e0a01406ad81e2076edcebf"}, - {file = "wrapt-2.1.1-cp311-cp311-win32.whl", hash = "sha256:9e60a30aa0909435ec4ea2a3c53e8e1b50ac9f640c0e9fe3f21fd248a22f06c5"}, - {file = "wrapt-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:7d79954f51fcf84e5ec4878ab4aea32610d70145c5bbc84b3370eabfb1e096c2"}, - {file = "wrapt-2.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:d3ffc6b0efe79e08fd947605fd598515aebefe45e50432dc3b5cd437df8b1ada"}, - {file = "wrapt-2.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab8e3793b239db021a18782a5823fcdea63b9fe75d0e340957f5828ef55fcc02"}, - {file = "wrapt-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7c0300007836373d1c2df105b40777986accb738053a92fe09b615a7a4547e9f"}, - {file = "wrapt-2.1.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2b27c070fd1132ab23957bcd4ee3ba707a91e653a9268dc1afbd39b77b2799f7"}, - {file = "wrapt-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b0e36d845e8b6f50949b6b65fc6cd279f47a1944582ed4ec8258cd136d89a64"}, - {file = "wrapt-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4aeea04a9889370fcfb1ef828c4cc583f36a875061505cd6cd9ba24d8b43cc36"}, - {file = "wrapt-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d88b46bb0dce9f74b6817bc1758ff2125e1ca9e1377d62ea35b6896142ab6825"}, - {file = "wrapt-2.1.1-cp312-cp312-win32.whl", hash = "sha256:63decff76ca685b5c557082dfbea865f3f5f6d45766a89bff8dc61d336348833"}, - {file = "wrapt-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:b828235d26c1e35aca4107039802ae4b1411be0fe0367dd5b7e4d90e562fcbcd"}, - {file = "wrapt-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:75128507413a9f1bcbe2db88fd18fbdbf80f264b82fa33a6996cdeaf01c52352"}, - {file = "wrapt-2.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ce9646e17fa7c3e2e7a87e696c7de66512c2b4f789a8db95c613588985a2e139"}, - {file = "wrapt-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:428cfc801925454395aa468ba7ddb3ed63dc0d881df7b81626cdd433b4e2b11b"}, - {file = "wrapt-2.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5797f65e4d58065a49088c3b32af5410751cd485e83ba89e5a45e2aa8905af98"}, - {file = "wrapt-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a2db44a71202c5ae4bb5f27c6d3afbc5b23053f2e7e78aa29704541b5dad789"}, - {file = "wrapt-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8d5350c3590af09c1703dd60ec78a7370c0186e11eaafb9dda025a30eee6492d"}, - {file = "wrapt-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d9b076411bed964e752c01b49fd224cc385f3a96f520c797d38412d70d08359"}, - {file = "wrapt-2.1.1-cp313-cp313-win32.whl", hash = "sha256:0bb7207130ce6486727baa85373503bf3334cc28016f6928a0fa7e19d7ecdc06"}, - {file = "wrapt-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:cbfee35c711046b15147b0ae7db9b976f01c9520e6636d992cd9e69e5e2b03b1"}, - {file = "wrapt-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:7d2756061022aebbf57ba14af9c16e8044e055c22d38de7bf40d92b565ecd2b0"}, - {file = "wrapt-2.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4814a3e58bc6971e46baa910ecee69699110a2bf06c201e24277c65115a20c20"}, - {file = "wrapt-2.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:106c5123232ab9b9f4903692e1fa0bdc231510098f04c13c3081f8ad71c3d612"}, - {file = "wrapt-2.1.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1a40b83ff2535e6e56f190aff123821eea89a24c589f7af33413b9c19eb2c738"}, - {file = "wrapt-2.1.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:789cea26e740d71cf1882e3a42bb29052bc4ada15770c90072cb47bf73fb3dbf"}, - {file = "wrapt-2.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ba49c14222d5e5c0ee394495a8655e991dc06cbca5398153aefa5ac08cd6ccd7"}, - {file = "wrapt-2.1.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ac8cda531fe55be838a17c62c806824472bb962b3afa47ecbd59b27b78496f4e"}, - {file = "wrapt-2.1.1-cp313-cp313t-win32.whl", hash = "sha256:b8af75fe20d381dd5bcc9db2e86a86d7fcfbf615383a7147b85da97c1182225b"}, - {file = "wrapt-2.1.1-cp313-cp313t-win_amd64.whl", hash = "sha256:45c5631c9b6c792b78be2d7352129f776dd72c605be2c3a4e9be346be8376d83"}, - {file = "wrapt-2.1.1-cp313-cp313t-win_arm64.whl", hash = "sha256:da815b9263947ac98d088b6414ac83507809a1d385e4632d9489867228d6d81c"}, - {file = "wrapt-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9aa1765054245bb01a37f615503290d4e207e3fd59226e78341afb587e9c1236"}, - {file = "wrapt-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:feff14b63a6d86c1eee33a57f77573649f2550935981625be7ff3cb7342efe05"}, - {file = "wrapt-2.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:81fc5f22d5fcfdbabde96bb3f5379b9f4476d05c6d524d7259dc5dfb501d3281"}, - {file = "wrapt-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:951b228ecf66def855d22e006ab9a1fc12535111ae7db2ec576c728f8ddb39e8"}, - {file = "wrapt-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ddf582a95641b9a8c8bd643e83f34ecbbfe1b68bc3850093605e469ab680ae3"}, - {file = "wrapt-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fc5c500966bf48913f795f1984704e6d452ba2414207b15e1f8c339a059d5b16"}, - {file = "wrapt-2.1.1-cp314-cp314-win32.whl", hash = "sha256:4aa4baadb1f94b71151b8e44a0c044f6af37396c3b8bcd474b78b49e2130a23b"}, - {file = "wrapt-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:860e9d3fd81816a9f4e40812f28be4439ab01f260603c749d14be3c0a1170d19"}, - {file = "wrapt-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3c59e103017a2c1ea0ddf589cbefd63f91081d7ce9d491d69ff2512bb1157e23"}, - {file = "wrapt-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9fa7c7e1bee9278fc4f5dd8275bc8d25493281a8ec6c61959e37cc46acf02007"}, - {file = "wrapt-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:39c35e12e8215628984248bd9c8897ce0a474be2a773db207eb93414219d8469"}, - {file = "wrapt-2.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:94ded4540cac9125eaa8ddf5f651a7ec0da6f5b9f248fe0347b597098f8ec14c"}, - {file = "wrapt-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da0af328373f97ed9bdfea24549ac1b944096a5a71b30e41c9b8b53ab3eec04a"}, - {file = "wrapt-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4ad839b55f0bf235f8e337ce060572d7a06592592f600f3a3029168e838469d3"}, - {file = "wrapt-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0d89c49356e5e2a50fa86b40e0510082abcd0530f926cbd71cf25bee6b9d82d7"}, - {file = "wrapt-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:f4c7dd22cf7f36aafe772f3d88656559205c3af1b7900adfccb70edeb0d2abc4"}, - {file = "wrapt-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f76bc12c583ab01e73ba0ea585465a41e48d968f6d1311b4daec4f8654e356e3"}, - {file = "wrapt-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:7ea74fc0bec172f1ae5f3505b6655c541786a5cabe4bbc0d9723a56ac32eb9b9"}, - {file = "wrapt-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e03b3d486eb39f5d3f562839f59094dcee30c4039359ea15768dc2214d9e07c"}, - {file = "wrapt-2.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0fdf3073f488ce4d929929b7799e3b8c52b220c9eb3f4a5a51e2dc0e8ff07881"}, - {file = "wrapt-2.1.1-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cb4f59238c6625fae2eeb72278da31c9cfba0ff4d9cbe37446b73caa0e9bcf7"}, - {file = "wrapt-2.1.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f794a1c148871b714cb566f5466ec8288e0148a1c417550983864b3981737cd"}, - {file = "wrapt-2.1.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:95ef3866631c6da9ce1fc0f1e17b90c4c0aa6d041fc70a11bc90733aee122e1a"}, - {file = "wrapt-2.1.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:66bc1b2446f01cbbd3c56b79a3a8435bcd4178ac4e06b091913f7751a7f528b8"}, - {file = "wrapt-2.1.1-cp39-cp39-win32.whl", hash = "sha256:1b9e08e57cabc32972f7c956d10e85093c5da9019faa24faf411e7dd258e528c"}, - {file = "wrapt-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:e75ad48c3cca739f580b5e14c052993eb644c7fa5b4c90aa51193280b30875ae"}, - {file = "wrapt-2.1.1-cp39-cp39-win_arm64.whl", hash = "sha256:9ccd657873b7f964711447d004563a2bc08d1476d7a1afcad310f3713e6f50f4"}, - {file = "wrapt-2.1.1-py3-none-any.whl", hash = "sha256:3b0f4629eb954394a3d7c7a1c8cca25f0b07cefe6aa8545e862e9778152de5b7"}, - {file = "wrapt-2.1.1.tar.gz", hash = "sha256:5fdcb09bf6db023d88f312bd0767594b414655d58090fc1c46b3414415f67fac"}, -] - -[package.extras] -dev = ["pytest", "setuptools"] +markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" +files = [ + {file = "wrapt-1.17.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88bbae4d40d5a46142e70d58bf664a89b6b4befaea7b2ecc14e03cedb8e06c04"}, + {file = "wrapt-1.17.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6b13af258d6a9ad602d57d889f83b9d5543acd471eee12eb51f5b01f8eb1bc2"}, + {file = "wrapt-1.17.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd341868a4b6714a5962c1af0bd44f7c404ef78720c7de4892901e540417111c"}, + {file = "wrapt-1.17.3-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f9b2601381be482f70e5d1051a5965c25fb3625455a2bf520b5a077b22afb775"}, + {file = "wrapt-1.17.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:343e44b2a8e60e06a7e0d29c1671a0d9951f59174f3709962b5143f60a2a98bd"}, + {file = "wrapt-1.17.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:33486899acd2d7d3066156b03465b949da3fd41a5da6e394ec49d271baefcf05"}, + {file = "wrapt-1.17.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e6f40a8aa5a92f150bdb3e1c44b7e98fb7113955b2e5394122fa5532fec4b418"}, + {file = "wrapt-1.17.3-cp310-cp310-win32.whl", hash = "sha256:a36692b8491d30a8c75f1dfee65bef119d6f39ea84ee04d9f9311f83c5ad9390"}, + {file = "wrapt-1.17.3-cp310-cp310-win_amd64.whl", hash = "sha256:afd964fd43b10c12213574db492cb8f73b2f0826c8df07a68288f8f19af2ebe6"}, + {file = "wrapt-1.17.3-cp310-cp310-win_arm64.whl", hash = "sha256:af338aa93554be859173c39c85243970dc6a289fa907402289eeae7543e1ae18"}, + {file = "wrapt-1.17.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:273a736c4645e63ac582c60a56b0acb529ef07f78e08dc6bfadf6a46b19c0da7"}, + {file = "wrapt-1.17.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5531d911795e3f935a9c23eb1c8c03c211661a5060aab167065896bbf62a5f85"}, + {file = "wrapt-1.17.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0610b46293c59a3adbae3dee552b648b984176f8562ee0dba099a56cfbe4df1f"}, + {file = "wrapt-1.17.3-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b32888aad8b6e68f83a8fdccbf3165f5469702a7544472bdf41f582970ed3311"}, + {file = "wrapt-1.17.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cccf4f81371f257440c88faed6b74f1053eef90807b77e31ca057b2db74edb1"}, + {file = "wrapt-1.17.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8a210b158a34164de8bb68b0e7780041a903d7b00c87e906fb69928bf7890d5"}, + {file = "wrapt-1.17.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:79573c24a46ce11aab457b472efd8d125e5a51da2d1d24387666cd85f54c05b2"}, + {file = "wrapt-1.17.3-cp311-cp311-win32.whl", hash = "sha256:c31eebe420a9a5d2887b13000b043ff6ca27c452a9a22fa71f35f118e8d4bf89"}, + {file = "wrapt-1.17.3-cp311-cp311-win_amd64.whl", hash = "sha256:0b1831115c97f0663cb77aa27d381237e73ad4f721391a9bfb2fe8bc25fa6e77"}, + {file = "wrapt-1.17.3-cp311-cp311-win_arm64.whl", hash = "sha256:5a7b3c1ee8265eb4c8f1b7d29943f195c00673f5ab60c192eba2d4a7eae5f46a"}, + {file = "wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0"}, + {file = "wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba"}, + {file = "wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd"}, + {file = "wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828"}, + {file = "wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9"}, + {file = "wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396"}, + {file = "wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc"}, + {file = "wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe"}, + {file = "wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c"}, + {file = "wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6"}, + {file = "wrapt-1.17.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a47681378a0439215912ef542c45a783484d4dd82bac412b71e59cf9c0e1cea0"}, + {file = "wrapt-1.17.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a30837587c6ee3cd1a4d1c2ec5d24e77984d44e2f34547e2323ddb4e22eb77"}, + {file = "wrapt-1.17.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:16ecf15d6af39246fe33e507105d67e4b81d8f8d2c6598ff7e3ca1b8a37213f7"}, + {file = "wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277"}, + {file = "wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d"}, + {file = "wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa"}, + {file = "wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050"}, + {file = "wrapt-1.17.3-cp313-cp313-win32.whl", hash = "sha256:53e5e39ff71b3fc484df8a522c933ea2b7cdd0d5d15ae82e5b23fde87d44cbd8"}, + {file = "wrapt-1.17.3-cp313-cp313-win_amd64.whl", hash = "sha256:1f0b2f40cf341ee8cc1a97d51ff50dddb9fcc73241b9143ec74b30fc4f44f6cb"}, + {file = "wrapt-1.17.3-cp313-cp313-win_arm64.whl", hash = "sha256:7425ac3c54430f5fc5e7b6f41d41e704db073309acfc09305816bc6a0b26bb16"}, + {file = "wrapt-1.17.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cf30f6e3c077c8e6a9a7809c94551203c8843e74ba0c960f4a98cd80d4665d39"}, + {file = "wrapt-1.17.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e228514a06843cae89621384cfe3a80418f3c04aadf8a3b14e46a7be704e4235"}, + {file = "wrapt-1.17.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5ea5eb3c0c071862997d6f3e02af1d055f381b1d25b286b9d6644b79db77657c"}, + {file = "wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b"}, + {file = "wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa"}, + {file = "wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7"}, + {file = "wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4"}, + {file = "wrapt-1.17.3-cp314-cp314-win32.whl", hash = "sha256:fbd3c8319de8e1dc79d346929cd71d523622da527cca14e0c1d257e31c2b8b10"}, + {file = "wrapt-1.17.3-cp314-cp314-win_amd64.whl", hash = "sha256:e1a4120ae5705f673727d3253de3ed0e016f7cd78dc463db1b31e2463e1f3cf6"}, + {file = "wrapt-1.17.3-cp314-cp314-win_arm64.whl", hash = "sha256:507553480670cab08a800b9463bdb881b2edeed77dc677b0a5915e6106e91a58"}, + {file = "wrapt-1.17.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ed7c635ae45cfbc1a7371f708727bf74690daedc49b4dba310590ca0bd28aa8a"}, + {file = "wrapt-1.17.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:249f88ed15503f6492a71f01442abddd73856a0032ae860de6d75ca62eed8067"}, + {file = "wrapt-1.17.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a03a38adec8066d5a37bea22f2ba6bbf39fcdefbe2d91419ab864c3fb515454"}, + {file = "wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e"}, + {file = "wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f"}, + {file = "wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056"}, + {file = "wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804"}, + {file = "wrapt-1.17.3-cp314-cp314t-win32.whl", hash = "sha256:41b1d2bc74c2cac6f9074df52b2efbef2b30bdfe5f40cb78f8ca22963bc62977"}, + {file = "wrapt-1.17.3-cp314-cp314t-win_amd64.whl", hash = "sha256:73d496de46cd2cdbdbcce4ae4bcdb4afb6a11234a1df9c085249d55166b95116"}, + {file = "wrapt-1.17.3-cp314-cp314t-win_arm64.whl", hash = "sha256:f38e60678850c42461d4202739f9bf1e3a737c7ad283638251e79cc49effb6b6"}, + {file = "wrapt-1.17.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:70d86fa5197b8947a2fa70260b48e400bf2ccacdcab97bb7de47e3d1e6312225"}, + {file = "wrapt-1.17.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:df7d30371a2accfe4013e90445f6388c570f103d61019b6b7c57e0265250072a"}, + {file = "wrapt-1.17.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:caea3e9c79d5f0d2c6d9ab96111601797ea5da8e6d0723f77eabb0d4068d2b2f"}, + {file = "wrapt-1.17.3-cp38-cp38-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:758895b01d546812d1f42204bd443b8c433c44d090248bf22689df673ccafe00"}, + {file = "wrapt-1.17.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02b551d101f31694fc785e58e0720ef7d9a10c4e62c1c9358ce6f63f23e30a56"}, + {file = "wrapt-1.17.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:656873859b3b50eeebe6db8b1455e99d90c26ab058db8e427046dbc35c3140a5"}, + {file = "wrapt-1.17.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:a9a2203361a6e6404f80b99234fe7fb37d1fc73487b5a78dc1aa5b97201e0f22"}, + {file = "wrapt-1.17.3-cp38-cp38-win32.whl", hash = "sha256:55cbbc356c2842f39bcc553cf695932e8b30e30e797f961860afb308e6b1bb7c"}, + {file = "wrapt-1.17.3-cp38-cp38-win_amd64.whl", hash = "sha256:ad85e269fe54d506b240d2d7b9f5f2057c2aa9a2ea5b32c66f8902f768117ed2"}, + {file = "wrapt-1.17.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:30ce38e66630599e1193798285706903110d4f057aab3168a34b7fdc85569afc"}, + {file = "wrapt-1.17.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:65d1d00fbfb3ea5f20add88bbc0f815150dbbde3b026e6c24759466c8b5a9ef9"}, + {file = "wrapt-1.17.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a7c06742645f914f26c7f1fa47b8bc4c91d222f76ee20116c43d5ef0912bba2d"}, + {file = "wrapt-1.17.3-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7e18f01b0c3e4a07fe6dfdb00e29049ba17eadbc5e7609a2a3a4af83ab7d710a"}, + {file = "wrapt-1.17.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f5f51a6466667a5a356e6381d362d259125b57f059103dd9fdc8c0cf1d14139"}, + {file = "wrapt-1.17.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:59923aa12d0157f6b82d686c3fd8e1166fa8cdfb3e17b42ce3b6147ff81528df"}, + {file = "wrapt-1.17.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:46acc57b331e0b3bcb3e1ca3b421d65637915cfcd65eb783cb2f78a511193f9b"}, + {file = "wrapt-1.17.3-cp39-cp39-win32.whl", hash = "sha256:3e62d15d3cfa26e3d0788094de7b64efa75f3a53875cdbccdf78547aed547a81"}, + {file = "wrapt-1.17.3-cp39-cp39-win_amd64.whl", hash = "sha256:1f23fa283f51c890eda8e34e4937079114c74b4c81d2b2f1f1d94948f5cc3d7f"}, + {file = "wrapt-1.17.3-cp39-cp39-win_arm64.whl", hash = "sha256:24c2ed34dc222ed754247a2702b1e1e89fdbaa4016f324b4b8f1a802d4ffe87f"}, + {file = "wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22"}, + {file = "wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0"}, +] [[package]] name = "xgrammar" @@ -8710,7 +10918,7 @@ description = "Efficient, Flexible and Portable Structured Generation" optional = true python-versions = "<4,>=3.8" groups = ["main"] -markers = "extra == \"vllm\" and (platform_machine == \"x86_64\" or platform_machine == \"aarch64\" or platform_machine == \"arm64\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\")" +markers = "(extra == \"rl\" or extra == \"async-rl\") and (platform_machine == \"x86_64\" or platform_machine == \"aarch64\" or platform_machine == \"arm64\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\")" files = [ {file = "xgrammar-0.1.29-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:fdc66e834b915cf956168ac086bd577f138261644b944e73d73f07085682a4d8"}, {file = "xgrammar-0.1.29-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:48c5a5c60c5ca5ab09ff5ef9f6b382384a04b153bae5908006cd4f7d80d71e07"}, @@ -9043,6 +11251,18 @@ idna = ">=2.0" multidict = ">=4.0" propcache = ">=0.2.1" +[[package]] +name = "zhconv" +version = "1.4.3" +description = "A simple implementation of Simplified-Traditional Chinese conversion." +optional = true +python-versions = "*" +groups = ["main"] +markers = "extra == \"eval\"" +files = [ + {file = "zhconv-1.4.3.tar.gz", hash = "sha256:ad42d9057ca0605f8e41d62b67ca797f879f58193ee6840562c51459b2698c45"}, +] + [[package]] name = "zipp" version = "3.23.0" @@ -9050,7 +11270,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"megatron\" or extra == \"ray\"" +markers = "(python_version >= \"3.11\" or extra == \"eval\" or extra == \"megatron\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\") and (python_version == \"3.11\" or extra == \"eval\" or extra == \"megatron\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\" or extra == \"client\") and (extra == \"client\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\" or extra == \"eval\" or extra == \"megatron\")" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, @@ -9064,16 +11284,131 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] +[[package]] +name = "zstandard" +version = "0.25.0" +description = "Zstandard bindings for Python" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.11\" and extra == \"client\"" +files = [ + {file = "zstandard-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd"}, + {file = "zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7"}, + {file = "zstandard-0.25.0-cp310-cp310-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550"}, + {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d"}, + {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b"}, + {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0"}, + {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0"}, + {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd"}, + {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701"}, + {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1"}, + {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150"}, + {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab"}, + {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e"}, + {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74"}, + {file = "zstandard-0.25.0-cp310-cp310-win32.whl", hash = "sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa"}, + {file = "zstandard-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e"}, + {file = "zstandard-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c"}, + {file = "zstandard-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f"}, + {file = "zstandard-0.25.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431"}, + {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a"}, + {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc"}, + {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6"}, + {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072"}, + {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277"}, + {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313"}, + {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097"}, + {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778"}, + {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065"}, + {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa"}, + {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7"}, + {file = "zstandard-0.25.0-cp311-cp311-win32.whl", hash = "sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4"}, + {file = "zstandard-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2"}, + {file = "zstandard-0.25.0-cp311-cp311-win_arm64.whl", hash = "sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137"}, + {file = "zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b"}, + {file = "zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00"}, + {file = "zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64"}, + {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea"}, + {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb"}, + {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a"}, + {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902"}, + {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f"}, + {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b"}, + {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6"}, + {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91"}, + {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708"}, + {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512"}, + {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa"}, + {file = "zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd"}, + {file = "zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01"}, + {file = "zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9"}, + {file = "zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94"}, + {file = "zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1"}, + {file = "zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f"}, + {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea"}, + {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e"}, + {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551"}, + {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a"}, + {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611"}, + {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3"}, + {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b"}, + {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851"}, + {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250"}, + {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98"}, + {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf"}, + {file = "zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09"}, + {file = "zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5"}, + {file = "zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049"}, + {file = "zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3"}, + {file = "zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f"}, + {file = "zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c"}, + {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439"}, + {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043"}, + {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859"}, + {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0"}, + {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7"}, + {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2"}, + {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344"}, + {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c"}, + {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088"}, + {file = "zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12"}, + {file = "zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2"}, + {file = "zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d"}, + {file = "zstandard-0.25.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b9af1fe743828123e12b41dd8091eca1074d0c1569cc42e6e1eee98027f2bbd0"}, + {file = "zstandard-0.25.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4b14abacf83dfb5c25eb4e4a79520de9e7e205f72c9ee7702f91233ae57d33a2"}, + {file = "zstandard-0.25.0-cp39-cp39-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:a51ff14f8017338e2f2e5dab738ce1ec3b5a851f23b18c1ae1359b1eecbee6df"}, + {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3b870ce5a02d4b22286cf4944c628e0f0881b11b3f14667c1d62185a99e04f53"}, + {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:05353cef599a7b0b98baca9b068dd36810c3ef0f42bf282583f438caf6ddcee3"}, + {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:19796b39075201d51d5f5f790bf849221e58b48a39a5fc74837675d8bafc7362"}, + {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:53e08b2445a6bc241261fea89d065536f00a581f02535f8122eba42db9375530"}, + {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1f3689581a72eaba9131b1d9bdbfe520ccd169999219b41000ede2fca5c1bfdb"}, + {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d8c56bb4e6c795fc77d74d8e8b80846e1fb8292fc0b5060cd8131d522974b751"}, + {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:53f94448fe5b10ee75d246497168e5825135d54325458c4bfffbaafabcc0a577"}, + {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:c2ba942c94e0691467ab901fc51b6f2085ff48f2eea77b1a48240f011e8247c7"}, + {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:07b527a69c1e1c8b5ab1ab14e2afe0675614a09182213f21a0717b62027b5936"}, + {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:51526324f1b23229001eb3735bc8c94f9c578b1bd9e867a0a646a3b17109f388"}, + {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89c4b48479a43f820b749df49cd7ba2dbc2b1b78560ecb5ab52985574fd40b27"}, + {file = "zstandard-0.25.0-cp39-cp39-win32.whl", hash = "sha256:1cd5da4d8e8ee0e88be976c294db744773459d51bb32f707a0f166e5ad5c8649"}, + {file = "zstandard-0.25.0-cp39-cp39-win_amd64.whl", hash = "sha256:37daddd452c0ffb65da00620afb8e17abd4adaae6ce6310702841760c2c26860"}, + {file = "zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b"}, +] + +[package.extras] +cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and python_version < \"3.14\"", "cffi (>=2.0.0b) ; platform_python_implementation != \"PyPy\" and python_version >= \"3.14\""] + [extras] +async-rl = ["TransferQueue", "math-verify", "ray", "swanlab", "vllm"] +client = ["httpx", "openai", "plotext", "textual", "tinker"] +data = ["py-data-juicer"] docs = ["docutils", "myst_parser", "recommonmark", "sphinx", "sphinx-book-theme", "sphinx-copybutton", "sphinx-rtd-theme", "sphinx_markdown_tables", "sphinxcontrib-mermaid"] -kernels = ["kernels"] -megatron = ["megatron-core", "transformer-engine"] -ray = ["ray"] -tinker = ["tinker"] -transformers = ["accelerate", "torch", "torchvision"] -vllm = ["vllm"] +eval = ["evalscope"] +megatron = ["mcore_bridge", "megatron-core", "transformer-engine"] +rl = ["ray", "vllm"] +server = ["opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-instrumentation-logging", "opentelemetry-sdk", "psutil", "pynvml", "redis"] +test = ["hypothesis", "pytest", "pytest-asyncio"] [metadata] lock-version = "2.1" -python-versions = ">=3.11,<3.13" -content-hash = "6bc839d412edaa773717488aaf25cb1e5b663f305e3f9945d9f92fe1524160d2" +python-versions = ">=3.10,<=3.13" +content-hash = "2d8b26118c2aa4ef455e5e6983c89df8abc28279fb59146b867ccb9d31441b02" diff --git a/pyproject.toml b/pyproject.toml index 4b52c9c9..3641f275 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ client = [ "plotext>=5.2.0", "openai>=1.0.0", "httpx>=0.25.0", - "tinker==0.16.1", + "tinker==0.29.0; python_version >= '3.11'", ] server = [ "redis>=5.0", diff --git a/src/twinkle/infra/__init__.py b/src/twinkle/infra/__init__.py index 2cf7c148..6d6ae8d0 100644 --- a/src/twinkle/infra/__init__.py +++ b/src/twinkle/infra/__init__.py @@ -10,7 +10,7 @@ from twinkle.notifier import Notifier, notify_exception from twinkle.utils import DeviceGroup, DeviceMesh, Platform, check_unsafe, framework_util, get_logger, requires -from .collectors import collect_tensor_dict +from .collectors import collect_tensor_dict as collect_tensor_dict logger = get_logger() @@ -530,7 +530,7 @@ def _run_continous_work(self, func_name: str, execute_method, workers, args, kwa try: ordered: List[Any] = [None] * batch_len for _, indices, ref in submitted: - part = ray.get(ref, timeout=ray_get_timeout) if ray_get_timeout else ray.get(ref) + part = ray.get(ref, timeout=ray_get_timeout) if ray_get_timeout is not None else ray.get(ref) if not isinstance(part, (list, tuple)) or len(part) != len(indices): raise TypeError(f'{func_name}: enable_continous_work needs one result per request, but a worker given ' f'{len(indices)} request(s) returned {type(part).__name__} of length ' @@ -740,7 +740,6 @@ def _get_device_mesh_param(args, kwargs): def _prepare_lazy_collect(args, kwargs): # if a worker received an actor handle, # lazy collect should be false to prevent any outer function receives an object ref - from ._ray import RayHelper if not os.environ.get('WORKER_NAME'): # If this is a driver return args, kwargs diff --git a/src/twinkle/server/gateway/tinker_handlers.py b/src/twinkle/server/gateway/tinker_handlers.py index 9ef98ce1..af2e8adb 100644 --- a/src/twinkle/server/gateway/tinker_handlers.py +++ b/src/twinkle/server/gateway/tinker_handlers.py @@ -19,6 +19,7 @@ from twinkle.hub import HubOperation from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager +from twinkle.server.utils.task_errors import error_payload_from_stored from twinkle.server.utils.task_queue import QueueState from twinkle.server.utils.validation import get_token_from_request from twinkle.utils.logger import get_logger @@ -119,8 +120,8 @@ async def retrieve_future(request: Request, } if status == 'failed': - result = record.get('result', {}) - return {'error': result.get('error', 'Unknown error'), 'category': result.get('category', 'Server')} + payload = error_payload_from_stored(record.get('result'), request_id=request_id) + return payload.model_dump(mode='json', exclude_none=True) result = record.get('result') if result is None: diff --git a/src/twinkle/server/model/app.py b/src/twinkle/server/model/app.py index 545f5a45..0b73cbc6 100644 --- a/src/twinkle/server/model/app.py +++ b/src/twinkle/server/model/app.py @@ -7,7 +7,8 @@ """ from __future__ import annotations -from fastapi import FastAPI, Request +import asyncio +from fastapi import FastAPI, HTTPException, Request from ray import serve from ray.serve.config import RequestRouterConfig from typing import Any @@ -131,30 +132,18 @@ async def __init__(self, from twinkle.server.data_plane import DataPlaneProxy self.data_plane = DataPlaneProxy(data_plane_url) self._replica_registered = False - # Health status bit, set when a post-timeout probe finds the model actor - # unreachable; auto-cleared by the next successful probe (R3#3). self._model_unhealthy = False + self._health_probe_task = None - # Initialize mixins - # ModelManagement opts into the Admission_Gate (collective mis-pairing + - # queue-bypassing _cleanup_adapter); SamplerManagement does not. + actors = getattr(self.model, '_actors', None) self._init_task_queue( queue_config, deployment_name='Model', enable_admission_gate=True, - on_backend_timeout=self._probe_after_timeout) - # Bound every ray.get on this backend by the effective execution timeout - # (applies to both sync=True and sync=False dispatch). T4.1. - self.model._ray_get_timeout = self._task_queue_config.effective_execution_timeout - # Inject the execution bounds used by state hygiene to compute the absolute - # survival TTL for non-terminal future records (T5.3). Collect_Width = - # world_size = len(self._actors). - _actors = getattr(self.model, '_actors', None) - self.state.set_execution_bounds( - queue_timeout=self._task_queue_config.queue_timeout, - execution_timeout=self._task_queue_config.effective_execution_timeout, - collect_width=len(_actors) if _actors else 1, + on_backend_timeout=self._probe_after_timeout, + collect_width=len(actors) if actors else 1, ) + self.model._ray_get_timeout = self._task_queue_config.effective_execution_timeout self._init_adapter_manager(**(adapter_config or {})) await self._register_replica_on_startup() # Note: countdown task is started lazily in _ensure_sticky() @@ -173,6 +162,7 @@ async def _register_replica_on_startup(self) -> None: """Register this replica's capacity before Ray Serve marks it ready.""" if not self._replica_registered: await self.state.register_replica(self.replica_id, self.max_loras) + await self.state.touch_replica_last_seen(self.replica_id) self._replica_registered = True @serve.multiplexed(max_num_models_per_replica=5) @@ -189,8 +179,9 @@ async def _on_request_start(self, request: Request) -> str: await self._ensure_sticky() await self.state.touch_replica_last_seen(self.replica_id) await self._ensure_state_cleanup_started() - token = get_token_from_request(request) - return token + if self._model_unhealthy: + raise HTTPException(status_code=503, detail='Model actors are unavailable') + return get_token_from_request(request) async def shutdown(self) -> None: """Explicit async cleanup — called via FastAPI shutdown event.""" @@ -198,18 +189,10 @@ async def shutdown(self) -> None: await self.state.unregister_replica(self.replica_id) except Exception: pass + await self.shutdown_task_queue() await self.data_plane.close() - async def check_model_health(self) -> dict: - """Probe model actors liveness via a lightweight ping. - - Returns a dict with 'healthy' (bool) and 'detail' (str). A successful probe - clears the unhealthy status bit; a failed probe sets it (R3#3). - - The ping goes through the Blocking_Call_Boundary with ``admit=False`` so it - never blocks the event loop yet never queues behind the Admission_Gate -- - the moment a probe matters most is while a call is stuck holding the gate. - """ + async def _run_model_health_probe(self) -> dict: try: result = await self.call_backend(self.model.ping, admit=False) if result is True: @@ -221,6 +204,13 @@ async def check_model_health(self) -> dict: self._model_unhealthy = True return {'healthy': False, 'detail': f'model actor unreachable: {e}'} + async def check_model_health(self) -> dict: + """Run one coalesced actor probe outside the event loop.""" + current = getattr(self, '_health_probe_task', None) + if current is None or current.done(): + self._health_probe_task = asyncio.create_task(self._run_model_health_probe()) + return await asyncio.shield(self._health_probe_task) + def mark_unhealthy(self) -> None: """Flag the deployment unhealthy; /healthz returns 503 until a probe recovers it.""" self._model_unhealthy = True @@ -232,7 +222,6 @@ async def _probe_after_timeout(self) -> None: async def _cleanup_adapter(self, adapter_name: str) -> None: if self.get_resource_info(adapter_name): - self.clear_resource_state(adapter_name) if self.train_mode == 'full': # No PEFT adapter to remove; restore clean base weights so the # next tenant does not inherit this tenant's trained weights. @@ -242,6 +231,7 @@ async def _cleanup_adapter(self, adapter_name: str) -> None: await self.call_backend(self.model.reload_initial_weights) else: await self.call_backend(self.model.remove_adapter, adapter_name) + self.clear_resource_state(adapter_name) self.unregister_resource(adapter_name) await self.state.unload_model(adapter_name) diff --git a/src/twinkle/server/model/backends/megatron_model.py b/src/twinkle/server/model/backends/megatron_model.py index fb3da3f2..15dd3470 100644 --- a/src/twinkle/server/model/backends/megatron_model.py +++ b/src/twinkle/server/model/backends/megatron_model.py @@ -11,7 +11,6 @@ """ import torch from tinker import types -from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union from twinkle import remote_class, remote_function from twinkle.data_format import InputFeature, Trajectory @@ -135,7 +134,7 @@ def forward_backward(self, *, inputs: InputFeature | list[InputFeature] | Trajec output = super().forward_backward(inputs=inputs, **kwargs) return to_cpu_safe_output(output) - @remote_function(collect='first', lazy_collect=False, timeout=10) + @remote_function(collect='first', lazy_collect=False, sync=True, timeout=4) def ping(self) -> bool: """Lightweight liveness probe for watchdog health checks.""" return True diff --git a/src/twinkle/server/model/backends/mock_model.py b/src/twinkle/server/model/backends/mock_model.py index b6a79bd3..c9bc12ff 100644 --- a/src/twinkle/server/model/backends/mock_model.py +++ b/src/twinkle/server/model/backends/mock_model.py @@ -240,7 +240,7 @@ def remove_adapter(self, adapter_name: str) -> None: def has_adapter(self, adapter_name: str) -> bool: return adapter_name in self._adapters - @remote_function(collect='first', lazy_collect=False, timeout=10) + @remote_function(collect='first', lazy_collect=False, sync=True, timeout=4) def ping(self) -> bool: """Lightweight liveness probe for watchdog health checks.""" return True diff --git a/src/twinkle/server/model/backends/transformers_model.py b/src/twinkle/server/model/backends/transformers_model.py index a2bcba83..ff709a69 100644 --- a/src/twinkle/server/model/backends/transformers_model.py +++ b/src/twinkle/server/model/backends/transformers_model.py @@ -13,7 +13,6 @@ (InputFeature/Trajectory-based I/O) via /twinkle/* endpoints. """ from tinker import types -from typing import List, Union from twinkle import remote_class, remote_function from twinkle.data_format import InputFeature, Trajectory @@ -111,7 +110,7 @@ def forward_backward(self, *, inputs: InputFeature | list[InputFeature] | Trajec output = super().forward_backward(inputs=inputs, **kwargs) return to_cpu_safe_output(output) - @remote_function(collect='first', lazy_collect=False, timeout=10) + @remote_function(collect='first', lazy_collect=False, sync=True, timeout=4) def ping(self) -> bool: """Lightweight liveness probe for watchdog health checks.""" return True diff --git a/src/twinkle/server/model/tinker_handlers.py b/src/twinkle/server/model/tinker_handlers.py index 7f34c2ae..d868a0d2 100644 --- a/src/twinkle/server/model/tinker_handlers.py +++ b/src/twinkle/server/model/tinker_handlers.py @@ -10,9 +10,8 @@ import traceback from collections.abc import Callable from fastapi import Depends, FastAPI, Request -from peft import LoraConfig from tinker import types -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING if TYPE_CHECKING: from .app import ModelManagement @@ -20,6 +19,7 @@ from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager from twinkle.server.exceptions import FullModeBusyError from twinkle.server.utils import get_template_for_model +from twinkle.server.utils.task_queue.types import UserTaskError from twinkle.utils.logger import get_logger logger = get_logger() @@ -45,17 +45,11 @@ async def _create_adapter(): try: # Validate lora_config against the deployment's train_mode up front. if self.is_full_mode and body.lora_config: - return types.RequestFailedResponse( - error='This deployment runs in full-parameter (exclusive) mode; do not pass ' - 'lora_config. Use create_full_training_client (or omit lora_config).', - category=types.RequestErrorCategory.User, - ) + raise UserTaskError('This deployment runs in full-parameter (exclusive) mode; do not pass ' + 'lora_config. Use create_full_training_client (or omit lora_config).') if (not self.is_full_mode) and (not body.lora_config): - return types.RequestFailedResponse( - error='This deployment runs in LoRA mode; lora_config is required. ' - 'Use create_lora_training_client.', - category=types.RequestErrorCategory.User, - ) + raise UserTaskError('This deployment runs in LoRA mode; lora_config is required. ' + 'Use create_lora_training_client.') # Exclusive full-parameter training: reject early (before touching # state) if another tenant already holds the deployment. if self.is_full_mode: @@ -75,7 +69,7 @@ async def _create_adapter(): await self.call_backend(self.model.set_optimizer, 'Adam', adapter_name=model_adapter) self.set_resource_state(adapter_name, 'grad_ready', False) else: - # TODO: Make LoraConfig more flexible + from peft import LoraConfig lora_cfg = LoraConfig(r=body.lora_config.rank, target_modules='all-linear') self.register_resource(adapter_name, token, session_id=body.session_id) await self.call_backend( @@ -89,17 +83,13 @@ async def _create_adapter(): training_run_manager.save(_model_id, body) return types.CreateModelResponse(model_id=_model_id) except FullModeBusyError as e: - # Nothing was registered yet (check runs before register_model). - return types.RequestFailedResponse(error=str(e), category=types.RequestErrorCategory.User) + raise UserTaskError(str(e)) from e except Exception: if _model_id: adapter_name = self.get_adapter_name(adapter_name=_model_id) await self._cleanup_adapter(adapter_name) logger.error(traceback.format_exc()) - return types.RequestFailedResponse( - error=traceback.format_exc(), - category=types.RequestErrorCategory.Server, - ) + raise return await self.schedule_task(_create_adapter, token=token, task_type='create_model') @@ -159,10 +149,7 @@ async def _do_forward(): ) except Exception: logger.error(traceback.format_exc()) - return types.RequestFailedResponse( - error=traceback.format_exc(), - category=types.RequestErrorCategory.Server, - ) + raise datum_list = body.forward_input.data input_tokens = sum(len(d.model_input.to_ints()) for d in datum_list) @@ -209,10 +196,7 @@ async def _do_forward_backward(): ) except Exception: logger.error(traceback.format_exc()) - return types.RequestFailedResponse( - error=traceback.format_exc(), - category=types.RequestErrorCategory.Server, - ) + raise datum_list = body.forward_backward_input.data input_tokens = sum(len(d.model_input.to_ints()) for d in datum_list) @@ -253,10 +237,7 @@ async def _do_optim(): return types.OptimStepResponse(metrics=metrics) except Exception: logger.error(traceback.format_exc()) - return types.RequestFailedResponse( - error=traceback.format_exc(), - category=types.RequestErrorCategory.Server, - ) + raise return await self.schedule_task(_do_optim, model_id=body.model_id, token=token, task_type='optim_step') @@ -286,10 +267,7 @@ async def _do_save(): return types.SaveWeightsResponse(path=tinker_path, type='save_weights') except Exception: logger.error(traceback.format_exc()) - return types.RequestFailedResponse( - error=traceback.format_exc(), - category=types.RequestErrorCategory.Server, - ) + raise return await self.schedule_task(_do_save, model_id=body.model_id, token=token, task_type='save_weights') @@ -334,10 +312,7 @@ async def _do_save_for_sampler(): path=tinker_path, sampling_session_id=sampling_session_id) except Exception: logger.error(traceback.format_exc()) - return types.RequestFailedResponse( - error=traceback.format_exc(), - category=types.RequestErrorCategory.Server, - ) + raise return await self.schedule_task( _do_save_for_sampler, model_id=body.model_id, token=token, task_type='save_weights_for_sampler') @@ -365,9 +340,6 @@ async def _do_load(): return types.LoadWeightsResponse(path=body.path, type='load_weights') except Exception: logger.error(traceback.format_exc()) - return types.RequestFailedResponse( - error=traceback.format_exc(), - category=types.RequestErrorCategory.Server, - ) + raise return await self.schedule_task(_do_load, model_id=body.model_id, token=token, task_type='load_weights') diff --git a/src/twinkle/server/model/twinkle_handlers.py b/src/twinkle/server/model/twinkle_handlers.py index 01eadbe6..c743c5d3 100644 --- a/src/twinkle/server/model/twinkle_handlers.py +++ b/src/twinkle/server/model/twinkle_handlers.py @@ -8,13 +8,11 @@ """ from __future__ import annotations -import asyncio import torch import traceback from collections.abc import Callable from fastapi import Depends, FastAPI, HTTPException, Request from pathlib import Path -from peft import LoraConfig from typing import TYPE_CHECKING, Any if TYPE_CHECKING: @@ -29,7 +27,6 @@ select_output_rows) from twinkle.server.utils.validation import get_session_id_from_request from twinkle.utils.logger import get_logger -from twinkle_client.common.serialize import deserialize_object logger = get_logger() @@ -620,11 +617,7 @@ async def _task(): checkpoint_manager.get_ckpt_dir(model_id=model_id_to_load, checkpoint_id=checkpoint_id)) else: checkpoint_dir = body.checkpoint_dir - # Run blocking upload in thread pool so the event loop is not blocked. - # async_upload is intentionally ignored here: the task queue + client polling - # already provide the fire-and-forget / wait semantics without holding the - # HTTP connection open for the full duration of the upload. - await asyncio.to_thread( + await self.call_backend( self.model.upload_to_hub, checkpoint_dir=checkpoint_dir, hub_model_id=body.hub_model_id, @@ -672,6 +665,9 @@ async def add_adapter_to_model( raise HTTPException(status_code=400, detail=str(exc)) async def _task(): + from peft import LoraConfig + + from twinkle_client.common.serialize import deserialize_object config = deserialize_object(body.config) extra_kwargs = body.model_extra or {} training_run_manager = create_training_run_manager(token, client_type='twinkle') @@ -735,6 +731,7 @@ async def apply_patch( adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) async def _task(): + from twinkle_client.common.serialize import deserialize_object self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} patch_cls = deserialize_object(body.patch_cls) @@ -756,6 +753,7 @@ async def add_metric( adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) async def _task(): + from twinkle_client.common.serialize import deserialize_object self.assert_resource_exists(adapter_name) extra_kwargs = body.model_extra or {} metric_cls = deserialize_object(body.metric_cls) diff --git a/src/twinkle/server/processor/twinkle_handlers.py b/src/twinkle/server/processor/twinkle_handlers.py index d9da305b..e8f8792c 100644 --- a/src/twinkle/server/processor/twinkle_handlers.py +++ b/src/twinkle/server/processor/twinkle_handlers.py @@ -23,7 +23,6 @@ from twinkle.server.telemetry.tracing import traced_operation from twinkle.server.utils.validation import get_session_id_from_request, get_token_from_request from twinkle.utils.logger import get_logger -from twinkle_client.common.serialize import deserialize_object logger = get_logger() @@ -61,6 +60,7 @@ async def create( _kwargs.pop('remote_group', None) _kwargs.pop('device_mesh', None) + from twinkle_client.common.serialize import deserialize_object resolved_kwargs = {} for key, value in _kwargs.items(): if isinstance(value, str) and value.startswith('pid:'): @@ -107,6 +107,7 @@ async def call( assert function is not None, f'`{function_name}` not found in {processor.__class__}' assert hasattr(function, '_execute'), f'Cannot call inner method of {processor.__class__}' + from twinkle_client.common.serialize import deserialize_object resolved_kwargs = {} for key, value in _kwargs.items(): if isinstance(value, str) and value.startswith('pid:'): diff --git a/src/twinkle/server/sampler/app.py b/src/twinkle/server/sampler/app.py index 892a66bd..3e7d0145 100644 --- a/src/twinkle/server/sampler/app.py +++ b/src/twinkle/server/sampler/app.py @@ -7,7 +7,6 @@ """ from __future__ import annotations -import asyncio from fastapi import FastAPI, Request from ray import serve from typing import Any @@ -103,12 +102,12 @@ def __init__(self, self.sampler_type = sampler_type self.model_id = model_id replica_context = serve.get_replica_context() - replica_id = replica_context.replica_id.unique_id + self.replica_id = replica_context.replica_id.unique_id sampler_kwargs: dict[str, Any] = { 'model_id': model_id, 'remote_group': self.device_group.name, - 'instance_id': replica_id, + 'instance_id': self.replica_id, } if sampler_type != 'mock': sampler_kwargs.update( @@ -127,17 +126,25 @@ def __init__(self, from twinkle.server.data_plane import DataPlaneProxy self.data_plane = DataPlaneProxy(data_plane_url) - # Initialize task queue mixin - self._init_task_queue(queue_config, deployment_name='Sampler') - # Bound every ray.get on this backend by the effective execution timeout - # (applies to both sync=True and sync=False dispatch). T4.1. + actors = getattr(self.sampler, '_actors', None) + self._init_task_queue( + queue_config, + deployment_name='Sampler', + collect_width=len(actors) if actors else 1, + ) self.sampler._ray_get_timeout = self._task_queue_config.effective_execution_timeout async def shutdown(self) -> None: - cancel_all = getattr(self.sampler, 'cancel_all_generations', None) - if callable(cancel_all): - await asyncio.to_thread(cancel_all) - await self.data_plane.close() + try: + cancel_all = getattr(self.sampler, 'cancel_all_generations', None) + if callable(cancel_all): + await self.call_backend(cancel_all) + finally: + try: + await self.state.unregister_replica(self.replica_id) + finally: + await self.shutdown_task_queue() + await self.data_plane.close() @serve.multiplexed(max_num_models_per_replica=5) async def _sticky_entry(self, sticky_key: str): @@ -149,9 +156,9 @@ async def _ensure_sticky(self): async def _on_request_start(self, request: Request) -> str: await self._ensure_sticky() + await self.state.touch_replica_last_seen(self.replica_id) await self._ensure_state_cleanup_started() - token = get_token_from_request(request) - return token + return get_token_from_request(request) def build_sampler_app(model_id: str, diff --git a/src/twinkle/server/sampler/tinker_handlers.py b/src/twinkle/server/sampler/tinker_handlers.py index 921e4fdc..02cef849 100644 --- a/src/twinkle/server/sampler/tinker_handlers.py +++ b/src/twinkle/server/sampler/tinker_handlers.py @@ -8,6 +8,7 @@ import os import traceback +import uuid from collections.abc import Callable from fastapi import Depends, FastAPI, Request from tinker import types @@ -19,11 +20,29 @@ from twinkle.data_format import SamplingParams from twinkle.server.checkpoint import create_checkpoint_manager from twinkle.server.utils import get_template_for_model +from twinkle.server.utils.task_queue.types import UserTaskError from twinkle.utils.logger import get_logger logger = get_logger() +def _sampled_sequence(*, stop_reason, tokens, logprobs): + return types.SampledSequence( + stop_reason=stop_reason, + sequence_id=uuid.uuid4().hex, + _tokens_list=tokens, + _logprobs_list=logprobs, + ) + + +def _sample_response(*, sequences, prompt_logprobs, topk_prompt_logprobs): + return types.SampleResponse( + sequences=sequences, + _prompt_logprobs_list=prompt_logprobs, + _topk_prompt_logprobs_list=topk_prompt_logprobs, + ) + + def _register_tinker_sampler_routes(app: FastAPI, self_fn: Callable[[], SamplerManagement]) -> None: """Register the tinker sampler route on the given FastAPI app. @@ -71,10 +90,7 @@ async def _do_sample(): # Base-model sampling is valid when no model_path was provided. if adapter_uri and not os.path.exists(adapter_uri): - return types.RequestFailedResponse( - error=f'Adapter URI {model_path} does not exist. Please check the model_path.', - category=types.RequestErrorCategory.User, - ) + raise UserTaskError(f'Adapter URI {model_path} does not exist. Please check the model_path.') # Convert tinker SamplingParams to twinkle SamplingParams if needed sampling_params = None @@ -120,22 +136,19 @@ async def _do_sample(): if flattened and len(flattened) == len(seq.logprobs): logprobs = flattened tinker_sequences.append( - types.SampledSequence( + _sampled_sequence( stop_reason=seq.stop_reason, tokens=list(seq.tokens), logprobs=logprobs, )) - return types.SampleResponse( + return _sample_response( sequences=tinker_sequences, prompt_logprobs=responses[0].prompt_logprobs, topk_prompt_logprobs=responses[0].topk_prompt_logprobs, ) except Exception: logger.error(traceback.format_exc()) - return types.RequestFailedResponse( - error=traceback.format_exc(), - category=types.RequestErrorCategory.Server, - ) + raise input_tokens = len(body.prompt.to_ints()) return await self.schedule_task( diff --git a/src/twinkle/server/sampler/twinkle_handlers.py b/src/twinkle/server/sampler/twinkle_handlers.py index 22515b4e..c5428b05 100644 --- a/src/twinkle/server/sampler/twinkle_handlers.py +++ b/src/twinkle/server/sampler/twinkle_handlers.py @@ -15,8 +15,6 @@ from fastapi.responses import StreamingResponse from typing import TYPE_CHECKING -from twinkle_client.common.serialize import deserialize_object - if TYPE_CHECKING: from .app import SamplerManagement @@ -24,8 +22,9 @@ import twinkle_client.types as types from twinkle.data_format import InputFeature, SamplingParams, Trajectory -from twinkle.server.telemetry.correlation import MODEL_ID, TOKEN_ID +from twinkle.server.telemetry.correlation import MODEL_ID from twinkle.server.telemetry.tracing import traced_operation +from twinkle.server.utils.task_errors import task_error_payload from twinkle.server.utils.validation import get_session_id_from_request from twinkle.utils.logger import get_logger from twinkle_client.common.json_utils import json_safe @@ -132,31 +131,55 @@ def _submission_states(value) -> list[dict]: return value if isinstance(value, list) else [value] -async def _await_generation( - sampler, - submission_id: str, -): - """Poll an admitted generation without occupying the sampler admission queue.""" - collected = False +async def _stream_queue(q, sentinel, request_id: str, total_timeout: float, single_get_timeout: float = 60.0): + loop = asyncio.get_running_loop() + start = loop.time() try: + while True: + remaining = total_timeout - (loop.time() - start) + if remaining <= 0: + payload = task_error_payload( + 'sample_stream exceeded the execution time bound', request_id=request_id, error_code=504) + yield json.dumps(payload) + '\n' + break + try: + item = await asyncio.wait_for( + loop.run_in_executor(None, q.get), timeout=min(single_get_timeout, remaining)) + except asyncio.TimeoutError: + payload = task_error_payload( + 'sample_stream timed out waiting for the next token', request_id=request_id, error_code=504) + yield json.dumps(payload) + '\n' + break + if item == sentinel: + break + if isinstance(item, Exception): + payload = task_error_payload(f'{type(item).__name__}: {item}', request_id=request_id, error_code=500) + yield json.dumps(payload) + '\n' + break + delta, reason = item + yield json.dumps({'delta': delta, 'finish_reason': reason}) + '\n' + finally: + try: + q.shutdown(force=True) + except Exception: + pass + + +async def _await_generation(service: SamplerManagement, submission_id: str, timeout: float): + """Poll one admitted generation through the backend boundary.""" + collected = False + + async def poll(): + nonlocal collected poll_interval = 0.01 while True: try: - states = _submission_states(await asyncio.to_thread(sampler.get_generation_status, submission_id)) + states = _submission_states(await service.call_backend(service.sampler.get_generation_status, + submission_id)) except Exception as error: - # A pending read-only actor call can be cancelled by Ray while - # the generation submitted just above remains alive. Treating - # that as a generation failure makes the finally block discard - # otherwise valid rollout work. Retry only Ray's explicit task - # cancellation; actor death and application errors must still - # propagate immediately. from ray.exceptions import TaskCancelledError if not isinstance(error, TaskCancelledError): raise - logger.warning( - 'Generation status poll was cancelled; retrying submission %s', - submission_id, - ) await asyncio.sleep(poll_interval) poll_interval = min(poll_interval * 1.5, 0.25) continue @@ -168,15 +191,19 @@ async def _await_generation( error = failed.get('error') or failed.get('status', 'unknown failure') raise RuntimeError(f'generation {submission_id} failed: {error}') if states and all(state.get('status') == 'completed' for state in states): - responses = await asyncio.to_thread(sampler.collect_generation, submission_id) + responses = await service.call_backend(service.sampler.collect_generation, submission_id) collected = True return responses await asyncio.sleep(poll_interval) poll_interval = min(poll_interval * 1.5, 0.25) + + try: + return await asyncio.wait_for(poll(), timeout=timeout) finally: if not collected: try: - await asyncio.to_thread(sampler.cancel_generation, submission_id) + await asyncio.wait_for( + service.call_backend(service.sampler.cancel_generation, submission_id), timeout=4.0) except Exception: logger.warning('Failed to cancel generation %s', submission_id, exc_info=True) @@ -315,7 +342,7 @@ async def sample_to_data_plane( submission_id = uuid.uuid4().hex async def _admit(): - await asyncio.to_thread( + await self.call_backend( self.sampler.submit_generation, submission_id, inputs, @@ -338,7 +365,7 @@ async def _admit(): task_type='sample_admission', )) - responses = await _await_generation(self.sampler, submission_id) + responses = await _await_generation(self, submission_id, self._task_queue_config.effective_execution_timeout) rows, tags = _build_rollout_rows_and_tags( _to_sample_response_models(responses), group_ids=body.group_ids, @@ -368,7 +395,7 @@ async def unload_adapter_paths( resolved_paths.append(adapter_path) unload = getattr(self.sampler, 'unload_adapter_paths', None) if unload is not None: - unload(resolved_paths) + await self.call_backend(unload, resolved_paths) return {'status': 'ok'} @app.post('/twinkle/set_template', response_model=types.SetTemplateResponse) @@ -407,6 +434,7 @@ async def apply_patch( body: types.ApplyPatchRequest, self: SamplerManagement = Depends(self_fn), ) -> None: + from twinkle_client.common.serialize import deserialize_object extra_kwargs = body.model_extra or {} patch_cls = deserialize_object(body.patch_cls) with traced_operation('sampler.apply_patch'): @@ -465,14 +493,16 @@ async def sample_stream( from .backends import STREAM_SENTINEL - q = Queue(maxsize=128) + request_id = f'req_{uuid.uuid4().hex}' actors = self.sampler._actors if not actors: - # No available sampler actor -> structured error, not an IndexError (R4#12). + async def _no_actor_generator(): - yield json.dumps({'error': 'No available sampler actor', 'category': 'Server'}) + '\n' + payload = task_error_payload('No available sampler actor', request_id=request_id, error_code=503) + yield json.dumps(payload) + '\n' return StreamingResponse(_no_actor_generator(), media_type='application/x-ndjson') + q = Queue(maxsize=128) actor = actors[0] actor.sample_stream_to_queue.remote( q, @@ -482,45 +512,12 @@ async def _no_actor_generator(): adapter_path=adapter_path, ) - # Two time bounds (R4#10-11): a per-get bound and a total-lifetime bound. - single_get_timeout = 60.0 - total_timeout = self._task_queue_config.effective_execution_timeout - - async def _stream_generator(): - loop = asyncio.get_event_loop() - start = loop.time() - try: - while True: - remaining = total_timeout - (loop.time() - start) - if remaining <= 0: - yield json.dumps({ - 'error': 'sample_stream exceeded the execution time bound', - 'category': 'Server' - }) + '\n' - break - try: - item = await asyncio.wait_for( - loop.run_in_executor(None, q.get), timeout=min(single_get_timeout, remaining)) - except asyncio.TimeoutError: - yield json.dumps({ - 'error': 'sample_stream timed out waiting for the next token', - 'category': 'Server' - }) + '\n' - break - if item == STREAM_SENTINEL: - break - if isinstance(item, Exception): - yield json.dumps({'error': str(item)}) + '\n' - break - delta, reason = item - yield json.dumps({'delta': delta, 'finish_reason': reason}) + '\n' - finally: - # The run_in_executor(None, q.get) thread is NOT cancelled when - # wait_for times out; shutting the ray Queue down makes the blocked - # get() raise so the thread exits and the pool returns to baseline. - try: - q.shutdown(force=True) - except Exception: - pass - - return StreamingResponse(_stream_generator(), media_type='application/x-ndjson') + return StreamingResponse( + _stream_queue( + q, + STREAM_SENTINEL, + request_id, + self._task_queue_config.effective_execution_timeout, + ), + media_type='application/x-ndjson', + ) diff --git a/src/twinkle/server/state/future_manager.py b/src/twinkle/server/state/future_manager.py index 9d3b3a3c..3a9bd917 100644 --- a/src/twinkle/server/state/future_manager.py +++ b/src/twinkle/server/state/future_manager.py @@ -29,6 +29,7 @@ def _future_record_transform( queue_state: str | None, queue_state_reason: str | None, replica_id: str | None, + absolute_deadline: float | None, now: str, ) -> dict | None: """Atomic transform body for :meth:`FutureManager.store_status`. @@ -57,6 +58,7 @@ def _future_record_transform( queue_state=queue_state, queue_state_reason=queue_state_reason, replica_id=replica_id, + absolute_deadline=absolute_deadline, created_at=now, updated_at=now, ) @@ -79,10 +81,7 @@ def _future_record_transform( class FutureManager(BaseManager[FutureRecord]): - """Manages async task futures / request statuses. - - Expiry is based on `updated_at` (falls back to `created_at`). - """ + """Manage future state, terminal retention, and immutable task deadlines.""" def __init__(self, backend: StateBackend, expiration_timeout: float) -> None: super().__init__(backend, 'future::', FutureRecord, expiration_timeout) @@ -99,6 +98,7 @@ async def store_status( queue_state: str | None = None, queue_state_reason: str | None = None, replica_id: str | None = None, + absolute_deadline: float | None = None, ) -> None: """Create or update a future record with the latest status. @@ -125,6 +125,7 @@ async def store_status( queue_state=queue_state, queue_state_reason=queue_state_reason, replica_id=replica_id, + absolute_deadline=absolute_deadline, now=now, ), ) @@ -136,14 +137,13 @@ async def cleanup_expired( cutoff_time: float, *, alive_replica_ids: set[str] | None = None, - absolute_ttl: float | None = None, ) -> int: """Expire future records without ever deleting a non-terminal one. Processing matrix (design §5.2): - | status | replica alive | over absolute_ttl | action | - |--------------|---------------|-------------------|-------------------| + | status | replica alive | past deadline | action | + |--------------|---------------|---------------|-------------------| | Terminal | — | ts < cutoff | delete | | non-Terminal | yes | no | keep (untouched) | | non-Terminal | yes | yes | write ``failed`` | @@ -153,9 +153,6 @@ async def cleanup_expired( cutoff_time: Unix timestamp; terminal records older than it are deleted. alive_replica_ids: replicas currently considered alive. ``None`` disables the orphan check (every non-terminal record is treated as owned). - absolute_ttl: seconds; a non-terminal record whose ``created_at`` is older - than this (regardless of ``updated_at``) is failed. ``None`` disables - the absolute-survival bound. Returns: Number of terminal records removed (records written ``failed`` are not @@ -188,20 +185,21 @@ async def cleanup_expired( replica_id=replica_id, ) continue - if absolute_ttl is not None: - created = self._parse_timestamp(record.created_at) - if (now - created) > absolute_ttl: - await self.store_status( - request_id, - 'failed', - record.model_id, - result=task_error_payload( - 'Task exceeded the absolute survival bound without reaching a terminal state.', - request_id=request_id, - error_code=500, - ), - replica_id=replica_id, - ) + deadline = record.absolute_deadline + if deadline is None: + deadline = self._parse_timestamp(record.created_at) + self.expiration_timeout + if now > deadline: + await self.store_status( + request_id, + 'failed', + record.model_id, + result=task_error_payload( + 'Task exceeded the absolute survival bound without reaching a terminal state.', + request_id=request_id, + error_code=500, + ), + replica_id=replica_id, + ) for request_id in expired_ids: await self.remove(request_id) diff --git a/src/twinkle/server/state/model_manager.py b/src/twinkle/server/state/model_manager.py index f4ccf034..58edf59b 100644 --- a/src/twinkle/server/state/model_manager.py +++ b/src/twinkle/server/state/model_manager.py @@ -118,9 +118,9 @@ async def get_alive_replica_ids(self, liveness_threshold: float) -> set[str]: last_seen = await self._replicas.get_all_last_seen() now = time.time() alive: set[str] = set() - for rid in registered: + for rid in set(registered) | set(last_seen): ls = last_seen.get(rid) - if ls is None or (now - ls) <= liveness_threshold: + if (ls is None and rid in registered) or (ls is not None and (now - ls) <= liveness_threshold): alive.add(rid) return alive diff --git a/src/twinkle/server/state/models.py b/src/twinkle/server/state/models.py index d3d3133b..7b11813a 100644 --- a/src/twinkle/server/state/models.py +++ b/src/twinkle/server/state/models.py @@ -56,9 +56,8 @@ class FutureRecord(BaseModel): result: Any = None queue_state: str | None = None queue_state_reason: str | None = None - # Replica that produced the record; filled at creation and never overwritten by a - # later status update. ``None`` (pre-upgrade record) means "ownership unknown" and - # is only subject to the absolute survival bound (see FutureManager.cleanup_expired). + # Replica ownership and deadline are fixed when the record is created. replica_id: str | None = None + absolute_deadline: float | None = None created_at: str = Field(default_factory=_now_iso) updated_at: str = Field(default_factory=_now_iso) diff --git a/src/twinkle/server/state/replica_registry.py b/src/twinkle/server/state/replica_registry.py index d287c1d0..d3eda8d8 100644 --- a/src/twinkle/server/state/replica_registry.py +++ b/src/twinkle/server/state/replica_registry.py @@ -1,15 +1,8 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Backend-backed registry of replica capacity. +"""Backend-backed registry of replica capacity and liveness. -Each entry persists to ``replica::::max_loras`` in the configured -:class:`StateBackend` (Redis or the actor-wrapped RayActorBackend), so every -Ray Serve worker sees one consistent view of the cluster's capacity even -though each worker holds its own ``ServerState`` instance. - -The registry knows *only* about declared capacity. The current loaded-model -count is derived by querying the persisted ``model::*`` records directly — -nothing here caches that count, so concurrent writes from different workers -cannot drift into an inconsistent local index. +Capacity and ``last_seen`` use separate keys so sampler liveness does not alter +the model-capacity data shape. """ from __future__ import annotations @@ -37,7 +30,7 @@ def _replica_id_from_key(key: str) -> str | None: class ReplicaRegistry: - """Read/write replica capacity through the shared :class:`StateBackend`.""" + """Read/write replica capacity and liveness through the shared backend.""" def __init__(self, backend: StateBackend) -> None: self._backend = backend diff --git a/src/twinkle/server/state/server_state.py b/src/twinkle/server/state/server_state.py index e59b215d..fc8014a0 100644 --- a/src/twinkle/server/state/server_state.py +++ b/src/twinkle/server/state/server_state.py @@ -39,10 +39,6 @@ LEASE_TTL = 30 # seconds — leader loses the lease after this without a renew LEASE_RENEW = 10 # seconds — must be < LEASE_TTL/2 so two missed renews still beat the TTL -# Integer multiple of (queue_timeout + resource-release bound) used as the absolute -# survival bound for a non-terminal record; the margin absorbs scheduling slack. -_ABSOLUTE_TTL_MULTIPLIER = 2 - def _renew_if_owner(current: str | None, *, owner: str) -> str | None: """``update_atomic`` transform: only re-write the lease if it is still mine.""" @@ -91,13 +87,6 @@ def __init__( self._cleanup_task: asyncio.Task | None = None self._cleanup_running = False - # Execution bounds injected by ModelManagement after _init_task_queue - # (see set_execution_bounds). Used to compute the absolute survival bound - # for non-terminal future records. All None => absolute bound disabled. - self._queue_timeout: float | None = None - self._effective_execution_timeout: float | None = None - self._collect_width: int | None = None - # Leader election + metrics-publish loop state. ``metrics_update_interval`` # is a typed parameter (a misspelled key now fails loudly rather than # being silently ignored); it controls how often the leader pushes counts @@ -301,6 +290,7 @@ async def store_future_status( queue_state: str | None = None, queue_state_reason: str | None = None, replica_id: str | None = None, + absolute_deadline: float | None = None, ) -> None: """Store task status with optional result. @@ -330,6 +320,7 @@ async def store_future_status( queue_state=queue_state, queue_state_reason=queue_state_reason, replica_id=replica_id, + absolute_deadline=absolute_deadline, ) # ----- Configuration Management ----- @@ -384,14 +375,8 @@ async def cleanup_expired_resources(self) -> dict[str, int]: samplings_removed = await self._sampling_mgr.cleanup_expired( cutoff_time, expired_session_ids=expired_session_ids) - # State hygiene for future records (design §5.2): protect non-terminal - # records owned by a live replica, fail orphans, and enforce the absolute - # survival bound. Alive set comes from ReplicaRegistry; absolute_ttl is - # computed only when the execution bounds were injected. alive_replica_ids = await self._model_mgr.get_alive_replica_ids(self.expiration_timeout) - absolute_ttl = self._absolute_survival_ttl() - futures_removed = await self._future_mgr.cleanup_expired( - cutoff_time, alive_replica_ids=alive_replica_ids, absolute_ttl=absolute_ttl) + futures_removed = await self._future_mgr.cleanup_expired(cutoff_time, alive_replica_ids=alive_replica_ids) return { 'sessions': sessions_removed, @@ -400,24 +385,6 @@ async def cleanup_expired_resources(self) -> dict[str, int]: 'futures': futures_removed, } - def set_execution_bounds(self, *, queue_timeout: float, execution_timeout: float, collect_width: int) -> None: - """Inject the execution bounds used to compute the absolute survival TTL. - - Called by ModelManagement after ``_init_task_queue``. ``ServerState`` is a - shared instance obtained via ``get_server_state()``, so these cannot be - constructor arguments; a setter is the injection path (design §5.2). - """ - self._queue_timeout = queue_timeout - self._effective_execution_timeout = execution_timeout - self._collect_width = collect_width - - def _absolute_survival_ttl(self) -> float | None: - """``k * (queue_timeout + Collect_Width * T)`` or ``None`` if not injected.""" - if (self._queue_timeout is None or self._effective_execution_timeout is None or self._collect_width is None): - return None - resource_release_bound = self._collect_width * self._effective_execution_timeout - return _ABSOLUTE_TTL_MULTIPLIER * (self._queue_timeout + resource_release_bound) - async def touch_replica_last_seen(self, replica_id: str) -> None: """Refresh a replica's liveness timestamp in the shared registry (R4#6).""" await self._model_mgr.touch_replica_last_seen(replica_id) diff --git a/src/twinkle/server/utils/task_errors.py b/src/twinkle/server/utils/task_errors.py index 7e977cf1..031f71cf 100644 --- a/src/twinkle/server/utils/task_errors.py +++ b/src/twinkle/server/utils/task_errors.py @@ -29,25 +29,27 @@ def task_error_payload( *, request_id: str, error_code: int = 500, - category: ErrorCategory = ErrorCategory.Server, + category: ErrorCategory | str = ErrorCategory.Server, traceback_text: str | None = None, ) -> dict[str, Any]: """Build an ``ErrorPayload`` and return it as a JSON-safe dict for storage. Traceback splitting and length trimming happen here so over-long text is never - written to State_Backend. A ``User`` category carries no traceback (R5#6). + written to State_Backend. A ``user`` category carries no traceback. """ - tb: str | None = None - if category != ErrorCategory.User and traceback_text: - tb = _trim_traceback(traceback_text) + if isinstance(category, str): + category = ErrorCategory(category.lower()) + tb = _trim_traceback(traceback_text) if category is ErrorCategory.Server and traceback_text else None + lines = str(error).splitlines() + summary = (lines[0] if lines else '')[:_ERROR_MAX] payload = ErrorPayload( - error=error[:_ERROR_MAX], + error=summary, category=category, error_code=error_code, request_id=request_id, traceback=tb, ) - return payload.model_dump(mode='json') + return payload.model_dump(mode='json', exclude_none=True) def error_payload_from_stored(stored: Any, *, request_id: str) -> ErrorPayload: @@ -58,8 +60,14 @@ def error_payload_from_stored(stored: Any, *, request_id: str) -> ErrorPayload: the caller-supplied value / ``Unknown`` so a rolling upgrade never raises ``pydantic.ValidationError``. """ - data = dict(stored) if isinstance(stored, Mapping) else {'error': str(stored)} + if isinstance(stored, Mapping): + data = dict(stored) + else: + data = {'error': 'Unknown error' if stored is None else str(stored)} data.setdefault('category', ErrorCategory.Unknown) data.setdefault('error_code', 500) data.setdefault('request_id', request_id) + category = str(data['category']).lower() + if category != ErrorCategory.Server.value: + data.pop('traceback', None) return ErrorPayload.model_validate(data) diff --git a/src/twinkle/server/utils/task_queue/__init__.py b/src/twinkle/server/utils/task_queue/__init__.py index 5c90d318..d77dfe18 100644 --- a/src/twinkle/server/utils/task_queue/__init__.py +++ b/src/twinkle/server/utils/task_queue/__init__.py @@ -12,12 +12,13 @@ from .config import TaskQueueConfig from .mixin import TaskQueueMixin from .rate_limiter import RateLimiter -from .types import QueuedTask, QueueState, TaskStatus +from .types import QueuedTask, QueueState, TaskStatus, UserTaskError from .worker import ComputeWorker __all__ = [ 'TaskStatus', 'QueueState', + 'UserTaskError', 'QueuedTask', 'TaskQueueConfig', 'TaskQueueMixin', diff --git a/src/twinkle/server/utils/task_queue/config.py b/src/twinkle/server/utils/task_queue/config.py index fdeeaf28..4de98e0d 100644 --- a/src/twinkle/server/utils/task_queue/config.py +++ b/src/twinkle/server/utils/task_queue/config.py @@ -11,8 +11,10 @@ from pydantic import BaseModel, ConfigDict, Field -# Substituted when execution_timeout is left at 0 ("no configured limit"). +# Finite bounds used when configuration omits a limit and by long-running methods. _ZERO_EXECUTION_TIMEOUT_FALLBACK: float = 3600.0 +_MAX_DECLARED_BACKEND_TIMEOUT: float = 3600.0 +_ABSOLUTE_TTL_MULTIPLIER: int = 2 class TaskQueueConfig(BaseModel): @@ -57,3 +59,9 @@ def effective_execution_timeout(self) -> float: if self.execution_timeout > 0: return self.execution_timeout return _ZERO_EXECUTION_TIMEOUT_FALLBACK + + def absolute_future_ttl(self, collect_width: int) -> float: + """Conservative lifetime for a non-terminal future record.""" + ray_timeout = max(self.effective_execution_timeout, _MAX_DECLARED_BACKEND_TIMEOUT) + resource_bound = max(1, collect_width) * ray_timeout + return _ABSOLUTE_TTL_MULTIPLIER * (self.queue_timeout + resource_bound) diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/utils/task_queue/mixin.py index 4260b893..83526c39 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/utils/task_queue/mixin.py @@ -8,6 +8,7 @@ from __future__ import annotations import asyncio +import contextlib import functools import time import traceback @@ -19,6 +20,7 @@ from twinkle.server.telemetry.middleware import get_task_metrics from twinkle.server.utils.task_errors import task_error_payload from twinkle.utils.logger import get_logger +from twinkle_client.types.errors import ErrorCategory from .config import TaskQueueConfig from .rate_limiter import RateLimiter from .types import BackendBusyError, QueuedTask, QueueState, TaskStatus @@ -59,6 +61,7 @@ def _init_task_queue( *, enable_admission_gate: bool = False, on_backend_timeout: Callable[[], Coroutine[Any, Any, None]] | None = None, + collect_width: int = 1, ) -> None: """Initialise the task queue, rate limiter, and compute worker. @@ -71,9 +74,9 @@ def _init_task_queue( does not (vllm sampler owns its own concurrency and the weight-update / generation mutual exclusion is covered by infra ``_cw_barrier``). - ``on_backend_timeout`` is an optional coroutine invoked once whenever a task - fails with a Ray_Get_Timeout / execution timeout, used by ModelManagement to - probe actor liveness (R3#2). + ``on_backend_timeout`` runs after a backend timeout. ``collect_width`` is the + number of actor results a backend call may collect and determines the persisted + future deadline. """ self._task_queue_config = config if config is not None else TaskQueueConfig() if self._task_queue_config.execution_timeout == 0: @@ -82,6 +85,7 @@ def _init_task_queue( '(deployment=%s).', self._task_queue_config.effective_execution_timeout, deployment_name or 'unknown') self._deployment_name = deployment_name self._task_metrics = get_task_metrics(deployment_name) if deployment_name else None + self._future_absolute_ttl = self._task_queue_config.absolute_future_ttl(collect_width) self._rate_limiter = RateLimiter( rps_limit=self._task_queue_config.rps_limit, @@ -102,62 +106,91 @@ def _init_task_queue( on_backend_timeout=on_backend_timeout, ) - # Blocking_Call_Boundary: a dedicated thread pool that moves every backend - # call off the event loop. Deliberately NOT max_workers=1 -- a call that - # leaks past its wait_for timeout keeps its thread; capping at one worker - # would let one leak block the whole queue forever. self._backend_executor = ThreadPoolExecutor(thread_name_prefix='twinkle-backend') - # per-replica Admission_Gate; opt-in per deployment. - self._backend_admission: asyncio.Semaphore | None = (asyncio.Semaphore(1) if enable_admission_gate else None) - + self._backend_probe_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='twinkle-backend-probe') + self._backend_admission: asyncio.Lock | None = asyncio.Lock() if enable_admission_gate else None + self._backend_poisoned = asyncio.Event() self._event_loop: asyncio.AbstractEventLoop | None = None + async def _acquire_backend_gate(self, gate: asyncio.Lock) -> None: + if self._backend_poisoned.is_set(): + raise BackendBusyError('This replica is waiting for a timed-out backend call to exit.') + if not gate.locked(): + await gate.acquire() + else: + acquire_task = asyncio.create_task(gate.acquire()) + poison_task = asyncio.create_task(self._backend_poisoned.wait()) + try: + done, _ = await asyncio.wait((acquire_task, poison_task), return_when=asyncio.FIRST_COMPLETED) + except asyncio.CancelledError: + acquire_task.cancel() + poison_task.cancel() + await asyncio.gather(acquire_task, poison_task, return_exceptions=True) + if acquire_task.done() and not acquire_task.cancelled() and acquire_task.result(): + gate.release() + raise + if poison_task in done and self._backend_poisoned.is_set(): + if acquire_task.done() and not acquire_task.cancelled() and acquire_task.result(): + gate.release() + else: + acquire_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await acquire_task + raise BackendBusyError('This replica is waiting for a timed-out backend call to exit.') + poison_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await poison_task + await acquire_task + if self._backend_poisoned.is_set(): + gate.release() + raise BackendBusyError('This replica is waiting for a timed-out backend call to exit.') + async def call_backend(self, fn: Callable[..., Any], /, *args: Any, admit: bool = True, **kwargs: Any) -> Any: - """The only place a Blocking_Backend_Call leaves the event loop. - - Covers every backend call in this replica process -- not just handlers. - "Does it enter Task_Queue" is deliberately NOT the exemption test: what this - boundary protects is event-loop responsiveness, orthogonal to queueing. - ``check_model_health()``'s ping, ``_cleanup_adapter()``'s two calls (driven - by a countdown, never queued) and the non-queued sampler endpoints are - exactly the calls that test would have missed. - - The Admission_Gate it takes when ``admit=True`` guards against collective - mis-pairing (a task issued to the same actors while a timed-out call is - still in flight) and against queue-bypassing paths (``_cleanup_adapter``) -- - NOT GPU parallelism (Ray already serialises calls per actor). ``admit=False`` - skips the gate: it is for liveness probes only, because the moment a probe - matters most is while a call is stuck, and that is exactly when the gate is - held by the stuck thread; a probe touches no collective. The sampler side - does not enable the gate -- infra ``_cw_barrier`` already covers it. - - When the gate is held by a leaked (timed-out) call, a new admitting call - fails fast with :class:`BackendBusyError` instead of queueing behind it. + """Run one backend call outside the event loop. + + Normal model calls serialize through the admission gate. If the awaiting + task times out while its thread is still running, the gate is poisoned: + waiters fail immediately until that thread exits. Health probes bypass the + gate and use a reserved executor thread. Sampler deployments disable the + gate because their backend owns request concurrency. """ loop = asyncio.get_running_loop() - sem = self._backend_admission if admit else None - if sem is None: - return await loop.run_in_executor(self._backend_executor, functools.partial(fn, *args, **kwargs)) - - # asyncio.Semaphore.acquire() does not yield when the gate is free, so this - # check-then-acquire is race-free for a Semaphore(1): if not locked here, - # acquire succeeds synchronously. - if sem.locked(): - raise BackendBusyError('This replica is waiting for a timed-out backend call to exit; ' - 'refusing to queue behind it.') - await sem.acquire() - - def _work() -> Any: - try: - return fn(*args, **kwargs) - finally: - # Release only when the thread truly finishes. A wait_for timeout - # cancels the awaiting coroutine but NOT this thread; releasing on - # cancellation would free the gate while the backend call is still - # in flight (Property 4). - loop.call_soon_threadsafe(sem.release) + gate = self._backend_admission if admit else None + if gate is not None: + await self._acquire_backend_gate(gate) + + executor = self._backend_executor if admit else self._backend_probe_executor + try: + concurrent_future = executor.submit(functools.partial(fn, *args, **kwargs)) + except Exception: + if gate is not None and gate.locked(): + gate.release() + raise + + if gate is not None: - return await loop.run_in_executor(self._backend_executor, _work) + def release_gate(_future) -> None: + + def release() -> None: + self._backend_poisoned.clear() + if gate.locked(): + gate.release() + + with contextlib.suppress(RuntimeError): + loop.call_soon_threadsafe(release) + + concurrent_future.add_done_callback(release_gate) + + try: + return await asyncio.wrap_future(concurrent_future, loop=loop) + except asyncio.CancelledError: + if gate is not None and concurrent_future.running(): + self._backend_poisoned.set() + raise + + def _future_deadline(self) -> float: + ttl = getattr(self, '_future_absolute_ttl', self._task_queue_config.absolute_future_ttl(1)) + return time.time() + ttl @staticmethod def _queue_key(model_id: str | None, token: str | None) -> str: @@ -186,7 +219,13 @@ async def _perform_preflight_checks( return None async def reject(error_msg: str, queue_state: str) -> dict[str, Any]: - error_payload = {'error': error_msg, 'category': 'User'} + error_code = 429 if queue_state == QueueState.PAUSED_RATE_LIMIT.value else 400 + error_payload = task_error_payload( + error_msg, + request_id=request_id, + error_code=error_code, + category=ErrorCategory.User, + ) if persist_failure: await self.state.store_future_status( request_id, @@ -272,6 +311,7 @@ async def _schedule_task( model_id, queue_state=QueueState.ACTIVE.value, replica_id=getattr(self, 'replica_id', None), + absolute_deadline=self._future_deadline(), ) queue_key = self._queue_key(model_id=model_id, token=token) @@ -423,6 +463,7 @@ async def schedule_background_task( model_id, queue_state=QueueState.ACTIVE.value, replica_id=getattr(self, 'replica_id', None), + absolute_deadline=self._future_deadline(), ) async def _run() -> None: @@ -501,5 +542,7 @@ async def shutdown_task_queue(self) -> None: await self._compute_worker.stop() # Do not wait on threads that may be leaked on a timed-out backend call. if getattr(self, '_backend_executor', None) is not None: - self._backend_executor.shutdown(wait=False) + self._backend_executor.shutdown(wait=False, cancel_futures=True) + if getattr(self, '_backend_probe_executor', None) is not None: + self._backend_probe_executor.shutdown(wait=False, cancel_futures=True) logger.debug('[TaskQueue] Task queue shutdown complete') diff --git a/src/twinkle/server/utils/task_queue/types.py b/src/twinkle/server/utils/task_queue/types.py index a7283dd9..c7f51462 100644 --- a/src/twinkle/server/utils/task_queue/types.py +++ b/src/twinkle/server/utils/task_queue/types.py @@ -26,13 +26,17 @@ class TaskStatus(Enum): RATE_LIMITED = 'rate_limited' # Task rejected due to rate limiting +class UserTaskError(ValueError): + """A queued operation rejected because of caller input or usage.""" + + class BackendBusyError(RuntimeError): """Raised when the per-replica Admission_Gate is held by a leaked backend call. A new backend call arriving while the gate is closed (its holder is a call that already exceeded ``asyncio.wait_for`` but whose executor thread has not yet returned) fails fast with this error instead of queueing behind it. The worker - maps it to ``ErrorPayload(category='Server', error_code=503)``. + maps it to ``ErrorPayload(category='server', error_code=503)``. """ diff --git a/src/twinkle/server/utils/task_queue/worker.py b/src/twinkle/server/utils/task_queue/worker.py index 121d7345..b1e276cf 100644 --- a/src/twinkle/server/utils/task_queue/worker.py +++ b/src/twinkle/server/utils/task_queue/worker.py @@ -12,14 +12,15 @@ import time import traceback from collections import deque -from typing import TYPE_CHECKING, Any, Deque +from typing import TYPE_CHECKING, Any, Callable, Deque from twinkle.server.telemetry.correlation import MODEL_ID, TOKEN_ID from twinkle.server.telemetry.tracing import traced_operation from twinkle.server.utils.task_errors import task_error_payload from twinkle.utils.logger import get_logger +from twinkle_client.types.errors import ErrorCategory from .config import TaskQueueConfig -from .types import BackendBusyError, QueuedTask, QueueState, TaskStatus +from .types import BackendBusyError, QueuedTask, QueueState, TaskStatus, UserTaskError if TYPE_CHECKING: from twinkle.server.state import ServerState @@ -153,6 +154,7 @@ async def _store_task_failed( queue_state_reason: str | None = None, *, error_code: int = 500, + category: ErrorCategory = ErrorCategory.Server, traceback_text: str | None = None, ) -> None: """Store FAILED status with a standardised ``ErrorPayload``.""" @@ -165,6 +167,7 @@ async def _store_task_failed( error, request_id=task.request_id, error_code=error_code, + category=category, traceback_text=traceback_text, ), queue_state=queue_state, @@ -280,6 +283,16 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue await self._on_backend_timeout() except Exception: logger.error(f'[ComputeWorker] backend-timeout probe failed:\n{traceback.format_exc(limit=3)}') + except UserTaskError as exc: + task_status = 'failed' + exec_time = time.monotonic() - exec_start + await self._store_task_failed( + task, + f'{type(exc).__name__}: {exc}', + QueueState.UNKNOWN.value, + error_code=400, + category=ErrorCategory.User, + ) except BackendBusyError as exc: task_status = 'failed' exec_time = time.monotonic() - exec_start diff --git a/src/twinkle/utils/nccl_safe.py b/src/twinkle/utils/nccl_safe.py index 214f3a17..a7c7eef0 100644 --- a/src/twinkle/utils/nccl_safe.py +++ b/src/twinkle/utils/nccl_safe.py @@ -55,11 +55,19 @@ def wrapper(self, *args, **kwargs): return func(self, *args, **kwargs) except Exception as exc: import traceback + rank = _global_rank() + context = f'twinkle backend method={func.__name__}, global_rank={rank}' + if hasattr(exc, 'add_note'): + exc.add_note(context) + elif exc.args: + exc.args = (f'{exc.args[0]} [{context}]', *exc.args[1:]) + else: + exc.args = (context, ) tb = traceback.format_exc() if len(tb) > _TRACEBACK_LIMIT: tb = tb[-_TRACEBACK_LIMIT:] logger.error('[nccl_safe_megatron] %s in %s on global rank %s:\n%s', - type(exc).__name__, func.__name__, _global_rank(), tb) + type(exc).__name__, func.__name__, rank, tb) raise return wrapper diff --git a/src/twinkle_client/types/errors.py b/src/twinkle_client/types/errors.py index 61685b1e..f1f4e3e1 100644 --- a/src/twinkle_client/types/errors.py +++ b/src/twinkle_client/types/errors.py @@ -4,10 +4,10 @@ Twinkle <-> tinker exception mapping (verified, kept here so a future new exception class can be lined up against its tinker counterpart): -- tinker ``RequestFailedError`` (``tinker/_exceptions.py:176-196``; carries +- Tinker 0.29.0 ``RequestFailedError`` (``tinker/_exceptions.py``; carries ``message`` / ``request_id`` / ``category``) is the "the task completed in a failed - terminal state" exception. Its ``category`` uses the same three values as - :class:`ErrorCategory` (``Unknown`` / ``Server`` / ``User``). + terminal state" exception. Its wire values are ``unknown`` / ``server`` / + ``user``, matching :class:`ErrorCategory`; legacy TitleCase values are normalized. ``twinkle_client/utils/patch_tinker.py`` shows two SDKs can coexist in one process, so a semantically-equal but differently-named exception must be lookup-able. @@ -15,7 +15,7 @@ class can be lined up against its tinker counterpart): from __future__ import annotations from enum import StrEnum -from pydantic import Field +from pydantic import Field, field_validator, model_validator from typing import Any, Literal, Optional from .base import ResponseModel @@ -29,9 +29,9 @@ class can be lined up against its tinker counterpart): class ErrorCategory(StrEnum): """Error attribution. Matches tinker's ``RequestErrorCategory``.""" - Unknown = 'Unknown' - Server = 'Server' - User = 'User' + Unknown = 'unknown' + Server = 'server' + User = 'user' class ErrorPayload(ResponseModel): @@ -53,3 +53,16 @@ class ErrorPayload(ResponseModel): request_id: str traceback: Optional[str] = Field(default=None, max_length=65536) details: Optional[list[dict[str, Any]]] = None + + @field_validator('category', mode='before') + @classmethod + def normalize_legacy_category(cls, value: Any) -> Any: + if isinstance(value, str): + return value.lower() + return value + + @model_validator(mode='after') + def traceback_is_server_only(self) -> 'ErrorPayload': + if self.traceback is not None and self.category is not ErrorCategory.Server: + raise ValueError('traceback is only valid for server errors') + return self diff --git a/tests/infra/test_ray_get_timeout.py b/tests/infra/test_ray_get_timeout.py index f9949eac..00a3b79c 100644 --- a/tests/infra/test_ray_get_timeout.py +++ b/tests/infra/test_ray_get_timeout.py @@ -10,9 +10,9 @@ ray = pytest.importorskip('ray') -import twinkle.infra as infra -from twinkle.infra._ray.ray_helper import RayHelper -from twinkle.infra import remote_function +import twinkle.infra as infra # noqa: E402 +from twinkle.infra import remote_function # noqa: E402 +from twinkle.infra._ray.ray_helper import RayHelper # noqa: E402 @ray.remote @@ -24,6 +24,14 @@ def slow(self, seconds: float): time.sleep(seconds) return seconds + def slow_batch(self, seconds: list[float]): + import time + time.sleep(seconds[0]) + return seconds + + def _twinkle_async_slow_batch(self, seconds: list[float]): + return self.slow_batch(seconds) + @pytest.fixture(scope='module', autouse=True) def _ray_and_ray_mode(): @@ -103,6 +111,19 @@ def slow(self, seconds): wrapped(driver, 2.0) +def test_continuous_work_timeout_zero_is_not_treated_as_falsy(): + + def slow_batch(self, seconds): + return seconds + + wrapped = remote_function( + dispatch='all', collect='first', timeout=0, enable_continous_work=True)(slow_batch) + driver = _make_driver() + driver._ray_get_timeout = 100.0 + with pytest.raises(ray.exceptions.GetTimeoutError): + wrapped(driver, [1.0]) + + def test_decorator_timeout_zero_is_not_treated_as_falsy(): """timeout=0 means 'time out immediately', not 'fall back to unbounded'.""" diff --git a/tests/server/README.md b/tests/server/README.md index 44a998a8..49adf2cb 100644 --- a/tests/server/README.md +++ b/tests/server/README.md @@ -20,4 +20,8 @@ backend's internals: the **dispatch path** (that a call reaches the backend via Tests that need to prove validation or NCCL behavior are the GPU-gated end-to-end tests under `tests/server/integration/test_nccl_safe_*_e2e.py` (run only with -`TWINKLE_TEST_GPU_E2E=1` against a real server). +`TWINKLE_TEST_GPU_E2E=1` against a real server configured with the test execution timeout). + +The contract suite covers all five apps and recursively snapshots request and response +types. Tinker compatibility assertions follow the 0.29.0 SDK wire values. The blocking +boundary suite uses a real serial Ray actor for the health-probe timing case. diff --git a/tests/server/contract/client_api_baseline.json b/tests/server/contract/client_api_baseline.json index 61f0c48a..243a0567 100644 --- a/tests/server/contract/client_api_baseline.json +++ b/tests/server/contract/client_api_baseline.json @@ -1,533 +1,3937 @@ { + "data_plane": { + "paths": { + "/twinkle/append": { + "POST": { + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "DataRef": { + "description": "Opaque reference to rows stored in the server-side TransferQueue.", + "properties": { + "fields": { + "items": { + "type": "string" + }, + "title": "Fields", + "type": "array" + }, + "kind": { + "default": "data", + "title": "Kind", + "type": "string" + }, + "num_tokens": { + "default": 0, + "title": "Num Tokens", + "type": "integer" + }, + "ref_id": { + "title": "Ref Id", + "type": "string" + }, + "size": { + "title": "Size", + "type": "integer" + } + }, + "required": [ + "ref_id", + "size" + ], + "title": "DataRef", + "type": "object" + } + }, + "properties": { + "ref": { + "$ref": "#/$defs/DataRef" + }, + "rows": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Rows", + "type": "array" + }, + "tags": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + } + }, + "required": [ + "ref", + "rows" + ], + "title": "DataAppendRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "append", + "path": [], + "query": [], + "response": { + "description": "Opaque reference to rows stored in the server-side TransferQueue.", + "properties": { + "fields": { + "items": { + "type": "string" + }, + "title": "Fields", + "type": "array" + }, + "kind": { + "default": "data", + "title": "Kind", + "type": "string" + }, + "num_tokens": { + "default": 0, + "title": "Num Tokens", + "type": "integer" + }, + "ref_id": { + "title": "Ref Id", + "type": "string" + }, + "size": { + "title": "Size", + "type": "integer" + } + }, + "required": [ + "ref_id", + "size" + ], + "title": "DataRef", + "type": "object" + }, + "responses": {}, + "statusCode": 200 + } + }, + "/twinkle/get": { + "POST": { + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "DataRef": { + "description": "Opaque reference to rows stored in the server-side TransferQueue.", + "properties": { + "fields": { + "items": { + "type": "string" + }, + "title": "Fields", + "type": "array" + }, + "kind": { + "default": "data", + "title": "Kind", + "type": "string" + }, + "num_tokens": { + "default": 0, + "title": "Num Tokens", + "type": "integer" + }, + "ref_id": { + "title": "Ref Id", + "type": "string" + }, + "size": { + "title": "Size", + "type": "integer" + } + }, + "required": [ + "ref_id", + "size" + ], + "title": "DataRef", + "type": "object" + } + }, + "properties": { + "fields": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Fields" + }, + "include_tags": { + "default": false, + "title": "Include Tags", + "type": "boolean" + }, + "ref": { + "$ref": "#/$defs/DataRef" + } + }, + "required": [ + "ref" + ], + "title": "DataGetRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "get", + "path": [], + "query": [], + "response": { + "properties": { + "rows": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Rows", + "type": "array" + }, + "tags": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Tags", + "type": "array" + } + }, + "required": [ + "rows" + ], + "title": "DataRowsResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 + } + }, + "/twinkle/put": { + "POST": { + "body": [ + { + "name": "body", + "required": true, + "schema": { + "properties": { + "kind": { + "default": "data", + "title": "Kind", + "type": "string" + }, + "rows": { + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Rows", + "type": "array" + }, + "tags": { + "anyOf": [ + { + "items": { + "additionalProperties": true, + "type": "object" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tags" + } + }, + "required": [ + "rows" + ], + "title": "DataPutRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "put", + "path": [], + "query": [], + "response": { + "description": "Opaque reference to rows stored in the server-side TransferQueue.", + "properties": { + "fields": { + "items": { + "type": "string" + }, + "title": "Fields", + "type": "array" + }, + "kind": { + "default": "data", + "title": "Kind", + "type": "string" + }, + "num_tokens": { + "default": 0, + "title": "Num Tokens", + "type": "integer" + }, + "ref_id": { + "title": "Ref Id", + "type": "string" + }, + "size": { + "title": "Size", + "type": "integer" + } + }, + "required": [ + "ref_id", + "size" + ], + "title": "DataRef", + "type": "object" + }, + "responses": {}, + "statusCode": 200 + } + }, + "/twinkle/release": { + "POST": { + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "DataRef": { + "description": "Opaque reference to rows stored in the server-side TransferQueue.", + "properties": { + "fields": { + "items": { + "type": "string" + }, + "title": "Fields", + "type": "array" + }, + "kind": { + "default": "data", + "title": "Kind", + "type": "string" + }, + "num_tokens": { + "default": 0, + "title": "Num Tokens", + "type": "integer" + }, + "ref_id": { + "title": "Ref Id", + "type": "string" + }, + "size": { + "title": "Size", + "type": "integer" + } + }, + "required": [ + "ref_id", + "size" + ], + "title": "DataRef", + "type": "object" + } + }, + "properties": { + "ref": { + "$ref": "#/$defs/DataRef" + } + }, + "required": [ + "ref" + ], + "title": "DataReleaseRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "release", + "path": [], + "query": [], + "response": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "responses": {}, + "statusCode": 200 + } + } + } + }, "gateway": { "paths": { "/asample": { "POST": { - "operationId": "asample_asample_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "DmelChunk": { + "additionalProperties": false, + "properties": { + "dmel": { + "format": "binary", + "title": "Dmel", + "type": "string" + }, + "type": { + "const": "dmel", + "default": "dmel", + "title": "Type", + "type": "string" + } + }, + "required": [ + "dmel" + ], + "title": "DmelChunk", + "type": "object" + }, + "EncodedTextChunk": { + "additionalProperties": false, + "properties": { + "tokens": { + "items": { + "type": "integer" + }, + "title": "Tokens", + "type": "array" + }, + "type": { + "const": "encoded_text", + "default": "encoded_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "tokens" + ], + "title": "EncodedTextChunk", + "type": "object" + }, + "ImageAssetPointerChunk": { + "additionalProperties": false, + "properties": { + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" + }, + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "location": { + "title": "Location", + "type": "string" + }, + "type": { + "const": "image_asset_pointer", + "default": "image_asset_pointer", + "title": "Type", + "type": "string" + } + }, + "required": [ + "format", + "location" + ], + "title": "ImageAssetPointerChunk", + "type": "object" + }, + "ImageChunk": { + "additionalProperties": false, + "properties": { + "data": { + "format": "binary", + "title": "Data", + "type": "string" + }, + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" + }, + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "type": { + "const": "image", + "default": "image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "data", + "format" + ], + "title": "ImageChunk", + "type": "object" + }, + "ModelInput": { + "additionalProperties": false, + "properties": { + "chunks": { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/EncodedTextChunk" + }, + { + "$ref": "#/$defs/ImageAssetPointerChunk" + }, + { + "$ref": "#/$defs/ImageChunk" + }, + { + "$ref": "#/$defs/DmelChunk" + } + ] + }, + "title": "Chunks", + "type": "array" + } + }, + "required": [ + "chunks" + ], + "title": "ModelInput", + "type": "object" + }, + "SamplingParams": { + "properties": { + "max_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Max Tokens" + }, + "seed": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seed" + }, + "stop": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Stop" + }, + "temperature": { + "default": 1, + "title": "Temperature", + "type": "number" + }, + "top_k": { + "default": -1, + "title": "Top K", + "type": "integer" + }, + "top_p": { + "default": 1, + "title": "Top P", + "type": "number" + } + }, + "title": "SamplingParams", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "base_model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Model" + }, + "model_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Path" + }, + "num_samples": { + "default": 1, + "title": "Num Samples", + "type": "integer" + }, + "prompt": { + "$ref": "#/$defs/ModelInput" + }, + "prompt_logprobs": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Prompt Logprobs" + }, + "record_stability_info": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Record Stability Info" + }, + "sampling_params": { + "$ref": "#/$defs/SamplingParams" + }, + "sampling_session_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sampling Session Id" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seq Id" + }, + "topk_prompt_logprobs": { + "default": 0, + "title": "Topk Prompt Logprobs", + "type": "integer" + }, + "topk_sample_logprobs": { + "default": 0, + "title": "Topk Sample Logprobs", + "type": "integer" + }, + "type": { + "const": "sample", + "default": "sample", + "title": "Type", + "type": "string" + } + }, + "required": [ + "prompt", + "sampling_params" + ], + "title": "SampleRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "asample", + "path": [], + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, "/chat/completions": { "POST": { - "operationId": "chat_completions_chat_completions_post", - "parameters": [], - "responses": [ - "200" - ] + "body": [], + "cookies": [], + "headers": [], + "operationId": "chat_completions", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/create_model": { "POST": { - "operationId": "create_model_create_model_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "LoraConfig": { + "additionalProperties": false, + "properties": { + "rank": { + "title": "Rank", + "type": "integer" + }, + "seed": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seed" + }, + "train_attn": { + "default": true, + "title": "Train Attn", + "type": "boolean" + }, + "train_mlp": { + "default": true, + "title": "Train Mlp", + "type": "boolean" + }, + "train_unembed": { + "default": true, + "title": "Train Unembed", + "type": "boolean" + } + }, + "required": [ + "rank" + ], + "title": "LoraConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "base_model": { + "title": "Base Model", + "type": "string" + }, + "lora_config": { + "anyOf": [ + { + "$ref": "#/$defs/LoraConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "model_seq_id": { + "title": "Model Seq Id", + "type": "integer" + }, + "session_id": { + "title": "Session Id", + "type": "string" + }, + "type": { + "const": "create_model", + "default": "create_model", + "title": "Type", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "session_id", + "model_seq_id", + "base_model" + ], + "title": "CreateModelRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "create_model", + "path": [], + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, "/create_sampling_session": { "POST": { - "operationId": "create_sampling_session_create_sampling_session_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "base_model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Model" + }, + "model_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Path" + }, + "sampling_session_seq_id": { + "title": "Sampling Session Seq Id", + "type": "integer" + }, + "session_id": { + "title": "Session Id", + "type": "string" + }, + "type": { + "const": "create_sampling_session", + "default": "create_sampling_session", + "title": "Type", + "type": "string" + } + }, + "required": [ + "session_id", + "sampling_session_seq_id" + ], + "title": "CreateSamplingSessionRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "create_sampling_session", + "path": [], + "query": [], + "response": { + "properties": { + "sampling_session_id": { + "title": "Sampling Session Id", + "type": "string" + }, + "type": { + "const": "create_sampling_session", + "default": "create_sampling_session", + "title": "Type", + "type": "string" + } + }, + "required": [ + "sampling_session_id" + ], + "title": "CreateSamplingSessionResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/create_session": { "POST": { - "operationId": "create_session_create_session_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "project_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Project Id" + }, + "sdk_version": { + "title": "Sdk Version", + "type": "string" + }, + "tags": { + "items": { + "type": "string" + }, + "title": "Tags", + "type": "array" + }, + "type": { + "const": "create_session", + "default": "create_session", + "title": "Type", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "title": "User Metadata" + } + }, + "required": [ + "tags", + "user_metadata", + "sdk_version" + ], + "title": "CreateSessionRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "create_session", + "path": [], + "query": [], + "response": { + "properties": { + "error_message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Error Message" + }, + "info_message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Info Message" + }, + "session_id": { + "title": "Session Id", + "type": "string" + }, + "type": { + "const": "create_session", + "default": "create_session", + "title": "Type", + "type": "string" + }, + "warning_message": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Warning Message" + } + }, + "required": [ + "session_id" + ], + "title": "CreateSessionResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/forward": { "POST": { - "operationId": "forward_forward_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "properties": { + "forward_input": { + "properties": { + "data": { + "items": { + "properties": { + "loss_fn_input_spans": { + "pythonType": "'Optional[ProvenanceSpans]'" + }, + "loss_fn_inputs": { + "pythonType": "'LossFnInputs'" + }, + "model_input": { + "pythonType": "'ModelInput'" + }, + "model_input_spans": { + "pythonType": "'Optional[ProvenanceSpans]'" + } + }, + "required": [ + "model_input" + ], + "type": "object" + }, + "type": "array" + }, + "loss_fn": { + "enum": [ + "cross_entropy", + "importance_sampling", + "ppo", + "cispo", + "dro" + ] + }, + "loss_fn_config": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "data", + "loss_fn" + ], + "type": "object" + }, + "model_id": { + "type": "string" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "forward_input", + "model_id" + ], + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "forward", + "path": [], + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, "/forward_backward": { "POST": { - "operationId": "forward_backward_forward_backward_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "properties": { + "forward_backward_input": { + "properties": { + "data": { + "items": { + "properties": { + "loss_fn_input_spans": { + "pythonType": "'Optional[ProvenanceSpans]'" + }, + "loss_fn_inputs": { + "pythonType": "'LossFnInputs'" + }, + "model_input": { + "pythonType": "'ModelInput'" + }, + "model_input_spans": { + "pythonType": "'Optional[ProvenanceSpans]'" + } + }, + "required": [ + "model_input" + ], + "type": "object" + }, + "type": "array" + }, + "loss_fn": { + "enum": [ + "cross_entropy", + "importance_sampling", + "ppo", + "cispo", + "dro" + ] + }, + "loss_fn_config": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "data", + "loss_fn" + ], + "type": "object" + }, + "model_id": { + "type": "string" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "forward_backward_input", + "model_id" + ], + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "forward_backward", + "path": [], + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, "/get_info": { "POST": { - "operationId": "get_info_get_info_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "model_id": { + "title": "Model Id", + "type": "string" + }, + "type": { + "const": "get_info", + "default": "get_info", + "title": "Type", + "type": "string" + } + }, + "required": [ + "model_id" + ], + "title": "GetInfoRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "get_info", + "path": [], + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, "/get_server_capabilities": { "GET": { - "operationId": "get_server_capabilities_get_server_capabilities_get", - "parameters": [], - "responses": [ - "200" - ] + "body": [], + "cookies": [], + "headers": [], + "operationId": "get_server_capabilities", + "path": [], + "query": [], + "response": { + "$defs": { + "SupportedModel": { + "description": "Information about a model supported by the server.", + "properties": { + "max_context_length": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Max Context Length" + }, + "model_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Name" + }, + "sampleable": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sampleable" + }, + "trainable": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Trainable" + } + }, + "title": "SupportedModel", + "type": "object" + } + }, + "description": "Response containing the server's supported models and capabilities.", + "properties": { + "supported_models": { + "items": { + "$ref": "#/$defs/SupportedModel" + }, + "title": "Supported Models", + "type": "array" + } + }, + "required": [ + "supported_models" + ], + "title": "GetServerCapabilitiesResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/healthz": { "GET": { - "operationId": "healthz_healthz_get", - "parameters": [], - "responses": [ - "200" - ] + "body": [], + "cookies": [], + "headers": [], + "operationId": "healthz", + "path": [], + "query": [], + "response": { + "properties": { + "status": { + "const": "ok", + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "HealthResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/load_weights": { "POST": { - "operationId": "load_weights_load_weights_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "base_model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Model" + }, + "model_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Id" + }, + "model_seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Seq Id" + }, + "optimizer": { + "title": "Optimizer", + "type": "boolean" + }, + "path": { + "title": "Path", + "type": "string" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seq Id" + }, + "session_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Session Id" + }, + "type": { + "const": "load_weights", + "default": "load_weights", + "title": "Type", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + }, + "weights_access_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Weights Access Token" + } + }, + "required": [ + "path", + "optimizer" + ], + "title": "LoadWeightsRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "load_weights", + "path": [], + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, "/models": { "GET": { - "operationId": "list_models_models_get", - "parameters": [], - "responses": [ - "200" - ] + "body": [], + "cookies": [], + "headers": [], + "operationId": "list_models", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/optim_step": { "POST": { - "operationId": "optim_step_optim_step_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "AdamParams": { + "additionalProperties": false, + "properties": { + "beta1": { + "default": 0.9, + "title": "Beta1", + "type": "number" + }, + "beta2": { + "default": 0.95, + "title": "Beta2", + "type": "number" + }, + "eps": { + "default": 1e-12, + "title": "Eps", + "type": "number" + }, + "grad_clip_norm": { + "default": 0.0, + "title": "Grad Clip Norm", + "type": "number" + }, + "learning_rate": { + "default": 0.0001, + "title": "Learning Rate", + "type": "number" + }, + "weight_decay": { + "default": 0.0, + "title": "Weight Decay", + "type": "number" + } + }, + "title": "AdamParams", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "adam_params": { + "$ref": "#/$defs/AdamParams" + }, + "model_id": { + "title": "Model Id", + "type": "string" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seq Id" + }, + "type": { + "const": "optim_step", + "default": "optim_step", + "title": "Type", + "type": "string" + } + }, + "required": [ + "adam_params", + "model_id" + ], + "title": "OptimStepRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "optim_step", + "path": [], + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, "/retrieve_future": { "POST": { - "operationId": "retrieve_future_retrieve_future_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "allow_metadata_only": { + "default": false, + "title": "Allow Metadata Only", + "type": "boolean" + }, + "request_id": { + "title": "Request Id", + "type": "string" + } + }, + "required": [ + "request_id" + ], + "title": "FutureRetrieveRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "retrieve_future", + "path": [], + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, "/save_weights": { "POST": { - "operationId": "save_weights_save_weights_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "model_id": { + "title": "Model Id", + "type": "string" + }, + "overwrite": { + "default": false, + "title": "Overwrite", + "type": "boolean" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seq Id" + }, + "ttl_seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Ttl Seconds" + }, + "type": { + "const": "save_weights", + "default": "save_weights", + "title": "Type", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "model_id" + ], + "title": "SaveWeightsRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "save_weights", + "path": [], + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, "/save_weights_for_sampler": { "POST": { - "operationId": "save_weights_for_sampler_save_weights_for_sampler_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "model_id": { + "title": "Model Id", + "type": "string" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "sampling_session_seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sampling Session Seq Id" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seq Id" + }, + "ttl_seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Ttl Seconds" + }, + "type": { + "const": "save_weights_for_sampler", + "default": "save_weights_for_sampler", + "title": "Type", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "model_id" + ], + "title": "SaveWeightsForSamplerRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "save_weights_for_sampler", + "path": [], + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, "/session_heartbeat": { "POST": { - "operationId": "session_heartbeat_session_heartbeat_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "session_id": { + "title": "Session Id", + "type": "string" + }, + "type": { + "const": "session_heartbeat", + "default": "session_heartbeat", + "title": "Type", + "type": "string" + } + }, + "required": [ + "session_id" + ], + "title": "SessionHeartbeatRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "session_heartbeat", + "path": [], + "query": [], + "response": { + "properties": { + "type": { + "const": "session_heartbeat", + "default": "session_heartbeat", + "title": "Type", + "type": "string" + } + }, + "title": "SessionHeartbeatResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/telemetry": { "POST": { - "operationId": "telemetry_telemetry_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "GenericEvent": { + "properties": { + "event": { + "enum": [ + "SESSION_START", + "SESSION_END", + "UNHANDLED_EXCEPTION", + "GENERIC_EVENT" + ], + "title": "Event", + "type": "string" + }, + "event_data": { + "additionalProperties": true, + "default": {}, + "title": "Event Data", + "type": "object" + }, + "event_id": { + "title": "Event Id", + "type": "string" + }, + "event_name": { + "title": "Event Name", + "type": "string" + }, + "event_session_index": { + "title": "Event Session Index", + "type": "integer" + }, + "severity": { + "enum": [ + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "title": "Severity", + "type": "string" + }, + "timestamp": { + "format": "date-time", + "title": "Timestamp", + "type": "string" + } + }, + "required": [ + "event", + "event_id", + "event_name", + "event_session_index", + "severity", + "timestamp" + ], + "title": "GenericEvent", + "type": "object" + }, + "SessionEndEvent": { + "properties": { + "duration": { + "title": "Duration", + "type": "string" + }, + "event": { + "enum": [ + "SESSION_START", + "SESSION_END", + "UNHANDLED_EXCEPTION", + "GENERIC_EVENT" + ], + "title": "Event", + "type": "string" + }, + "event_id": { + "title": "Event Id", + "type": "string" + }, + "event_session_index": { + "title": "Event Session Index", + "type": "integer" + }, + "severity": { + "enum": [ + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "title": "Severity", + "type": "string" + }, + "timestamp": { + "format": "date-time", + "title": "Timestamp", + "type": "string" + } + }, + "required": [ + "duration", + "event", + "event_id", + "event_session_index", + "severity", + "timestamp" + ], + "title": "SessionEndEvent", + "type": "object" + }, + "SessionStartEvent": { + "properties": { + "event": { + "enum": [ + "SESSION_START", + "SESSION_END", + "UNHANDLED_EXCEPTION", + "GENERIC_EVENT" + ], + "title": "Event", + "type": "string" + }, + "event_id": { + "title": "Event Id", + "type": "string" + }, + "event_session_index": { + "title": "Event Session Index", + "type": "integer" + }, + "severity": { + "enum": [ + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "title": "Severity", + "type": "string" + }, + "timestamp": { + "format": "date-time", + "title": "Timestamp", + "type": "string" + } + }, + "required": [ + "event", + "event_id", + "event_session_index", + "severity", + "timestamp" + ], + "title": "SessionStartEvent", + "type": "object" + }, + "UnhandledExceptionEvent": { + "properties": { + "error_message": { + "title": "Error Message", + "type": "string" + }, + "error_type": { + "title": "Error Type", + "type": "string" + }, + "event": { + "enum": [ + "SESSION_START", + "SESSION_END", + "UNHANDLED_EXCEPTION", + "GENERIC_EVENT" + ], + "title": "Event", + "type": "string" + }, + "event_id": { + "title": "Event Id", + "type": "string" + }, + "event_session_index": { + "title": "Event Session Index", + "type": "integer" + }, + "severity": { + "enum": [ + "DEBUG", + "INFO", + "WARNING", + "ERROR", + "CRITICAL" + ], + "title": "Severity", + "type": "string" + }, + "timestamp": { + "format": "date-time", + "title": "Timestamp", + "type": "string" + }, + "traceback": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Traceback" + } + }, + "required": [ + "error_message", + "error_type", + "event", + "event_id", + "event_session_index", + "severity", + "timestamp" + ], + "title": "UnhandledExceptionEvent", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "events": { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/SessionStartEvent" + }, + { + "$ref": "#/$defs/SessionEndEvent" + }, + { + "$ref": "#/$defs/UnhandledExceptionEvent" + }, + { + "$ref": "#/$defs/GenericEvent" + } + ] + }, + "title": "Events", + "type": "array" + }, + "platform": { + "title": "Platform", + "type": "string" + }, + "process_uuid": { + "title": "Process Uuid", + "type": "string" + }, + "sdk_version": { + "title": "Sdk Version", + "type": "string" + }, + "session_id": { + "title": "Session Id", + "type": "string" + } + }, + "required": [ + "events", + "platform", + "sdk_version", + "session_id", + "process_uuid" + ], + "title": "TelemetrySendRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "telemetry", + "path": [], + "query": [], + "response": { + "properties": { + "status": { + "const": "accepted", + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "TelemetryResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/training_runs": { "GET": { - "operationId": "get_training_runs_training_runs_get", - "parameters": [ + "body": [], + "cookies": [], + "headers": [], + "operationId": "get_training_runs", + "path": [], + "query": [ { - "in": "query", "name": "limit", "required": false, "schema": { - "default": 20, - "title": "Limit", "type": "integer" } }, { - "in": "query", "name": "offset", "required": false, "schema": { - "default": 0, - "title": "Offset", "type": "integer" } } ], - "responses": [ - "200", - "422" - ] + "response": { + "$defs": { + "Checkpoint": { + "properties": { + "checkpoint_id": { + "title": "Checkpoint Id", + "type": "string" + }, + "checkpoint_type": { + "enum": [ + "training", + "sampler", + "external" + ], + "title": "Checkpoint Type", + "type": "string" + }, + "expires_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expires At" + }, + "public": { + "default": false, + "title": "Public", + "type": "boolean" + }, + "size_bytes": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Size Bytes" + }, + "time": { + "format": "date-time", + "title": "Time", + "type": "string" + }, + "tinker_path": { + "title": "Tinker Path", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "checkpoint_id", + "checkpoint_type", + "time", + "tinker_path" + ], + "title": "Checkpoint", + "type": "object" + }, + "Cursor": { + "properties": { + "limit": { + "title": "Limit", + "type": "integer" + }, + "offset": { + "title": "Offset", + "type": "integer" + }, + "total_count": { + "title": "Total Count", + "type": "integer" + } + }, + "required": [ + "offset", + "limit", + "total_count" + ], + "title": "Cursor", + "type": "object" + }, + "TrainingRun": { + "properties": { + "base_model": { + "title": "Base Model", + "type": "string" + }, + "corrupted": { + "default": false, + "title": "Corrupted", + "type": "boolean" + }, + "is_lora": { + "title": "Is Lora", + "type": "boolean" + }, + "last_checkpoint": { + "anyOf": [ + { + "$ref": "#/$defs/Checkpoint" + }, + { + "type": "null" + } + ], + "default": null + }, + "last_request_time": { + "format": "date-time", + "title": "Last Request Time", + "type": "string" + }, + "last_sampler_checkpoint": { + "anyOf": [ + { + "$ref": "#/$defs/Checkpoint" + }, + { + "type": "null" + } + ], + "default": null + }, + "lora_rank": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lora Rank" + }, + "model_owner": { + "title": "Model Owner", + "type": "string" + }, + "training_run_id": { + "title": "Training Run Id", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "training_run_id", + "base_model", + "model_owner", + "is_lora", + "last_request_time" + ], + "title": "TrainingRun", + "type": "object" + } + }, + "properties": { + "cursor": { + "$ref": "#/$defs/Cursor" + }, + "training_runs": { + "items": { + "$ref": "#/$defs/TrainingRun" + }, + "title": "Training Runs", + "type": "array" + } + }, + "required": [ + "training_runs", + "cursor" + ], + "title": "TrainingRunsResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/training_runs/{run_id}": { "GET": { - "operationId": "get_training_run_training_runs__run_id__get", - "parameters": [ + "body": [], + "cookies": [], + "headers": [], + "operationId": "get_training_run", + "path": [ { - "in": "path", "name": "run_id", "required": true, "schema": { - "title": "Run Id", "type": "string" } } ], - "responses": [ - "200", - "422" - ] + "query": [], + "response": { + "$defs": { + "Checkpoint": { + "properties": { + "checkpoint_id": { + "title": "Checkpoint Id", + "type": "string" + }, + "checkpoint_type": { + "enum": [ + "training", + "sampler", + "external" + ], + "title": "Checkpoint Type", + "type": "string" + }, + "expires_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expires At" + }, + "public": { + "default": false, + "title": "Public", + "type": "boolean" + }, + "size_bytes": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Size Bytes" + }, + "time": { + "format": "date-time", + "title": "Time", + "type": "string" + }, + "tinker_path": { + "title": "Tinker Path", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "checkpoint_id", + "checkpoint_type", + "time", + "tinker_path" + ], + "title": "Checkpoint", + "type": "object" + } + }, + "properties": { + "base_model": { + "title": "Base Model", + "type": "string" + }, + "corrupted": { + "default": false, + "title": "Corrupted", + "type": "boolean" + }, + "is_lora": { + "title": "Is Lora", + "type": "boolean" + }, + "last_checkpoint": { + "anyOf": [ + { + "$ref": "#/$defs/Checkpoint" + }, + { + "type": "null" + } + ], + "default": null + }, + "last_request_time": { + "format": "date-time", + "title": "Last Request Time", + "type": "string" + }, + "last_sampler_checkpoint": { + "anyOf": [ + { + "$ref": "#/$defs/Checkpoint" + }, + { + "type": "null" + } + ], + "default": null + }, + "lora_rank": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lora Rank" + }, + "model_owner": { + "title": "Model Owner", + "type": "string" + }, + "training_run_id": { + "title": "Training Run Id", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "training_run_id", + "base_model", + "model_owner", + "is_lora", + "last_request_time" + ], + "title": "TrainingRun", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/training_runs/{run_id}/checkpoints": { "GET": { - "operationId": "get_run_checkpoints_training_runs__run_id__checkpoints_get", - "parameters": [ + "body": [], + "cookies": [], + "headers": [], + "operationId": "get_run_checkpoints", + "path": [ { - "in": "path", "name": "run_id", "required": true, "schema": { - "title": "Run Id", "type": "string" } } ], - "responses": [ - "200", - "422" - ] + "query": [], + "response": { + "$defs": { + "Checkpoint": { + "properties": { + "checkpoint_id": { + "title": "Checkpoint Id", + "type": "string" + }, + "checkpoint_type": { + "enum": [ + "training", + "sampler", + "external" + ], + "title": "Checkpoint Type", + "type": "string" + }, + "expires_at": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expires At" + }, + "public": { + "default": false, + "title": "Public", + "type": "boolean" + }, + "size_bytes": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Size Bytes" + }, + "time": { + "format": "date-time", + "title": "Time", + "type": "string" + }, + "tinker_path": { + "title": "Tinker Path", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "checkpoint_id", + "checkpoint_type", + "time", + "tinker_path" + ], + "title": "Checkpoint", + "type": "object" + }, + "Cursor": { + "properties": { + "limit": { + "title": "Limit", + "type": "integer" + }, + "offset": { + "title": "Offset", + "type": "integer" + }, + "total_count": { + "title": "Total Count", + "type": "integer" + } + }, + "required": [ + "offset", + "limit", + "total_count" + ], + "title": "Cursor", + "type": "object" + } + }, + "properties": { + "checkpoints": { + "items": { + "$ref": "#/$defs/Checkpoint" + }, + "title": "Checkpoints", + "type": "array" + }, + "cursor": { + "anyOf": [ + { + "$ref": "#/$defs/Cursor" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "checkpoints" + ], + "title": "CheckpointsListResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, - "/training_runs/{run_id}/checkpoints/{checkpoint_id}": { + "/training_runs/{run_id}/checkpoints/{checkpoint_id:path}": { "DELETE": { - "operationId": "delete_run_checkpoint_training_runs__run_id__checkpoints__checkpoint_id__delete", - "parameters": [ + "body": [], + "cookies": [], + "headers": [], + "operationId": "delete_run_checkpoint", + "path": [ { - "in": "path", "name": "run_id", "required": true, "schema": { - "title": "Run Id", "type": "string" } }, { - "in": "path", "name": "checkpoint_id", "required": true, "schema": { - "title": "Checkpoint Id", "type": "string" } } ], - "responses": [ - "200", - "422" - ] + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, - "/training_runs/{run_id}/checkpoints/{checkpoint_id}/publish": { + "/training_runs/{run_id}/checkpoints/{checkpoint_id:path}/publish": { "POST": { - "operationId": "publish_checkpoint_training_runs__run_id__checkpoints__checkpoint_id__publish_post", - "parameters": [ + "body": [], + "cookies": [], + "headers": [], + "operationId": "publish_checkpoint", + "path": [ { - "in": "path", "name": "run_id", "required": true, "schema": { - "title": "Run Id", "type": "string" } }, { - "in": "path", "name": "checkpoint_id", "required": true, "schema": { - "title": "Checkpoint Id", "type": "string" } } ], - "responses": [ - "200", - "422" - ] + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/capacity_info": { "GET": { - "operationId": "get_capacity_info_twinkle_capacity_info_get", - "parameters": [], - "responses": [ - "200" - ] + "body": [], + "cookies": [], + "headers": [], + "operationId": "get_capacity_info", + "path": [], + "query": [], + "response": { + "description": "Response body for the /capacity_info endpoint.", + "properties": { + "free_loras": { + "title": "Free Loras", + "type": "integer" + }, + "max_loras": { + "title": "Max Loras", + "type": "integer" + }, + "used_loras": { + "title": "Used Loras", + "type": "integer" + } + }, + "required": [ + "max_loras", + "used_loras", + "free_loras" + ], + "title": "CapacityInfoResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, - "/twinkle/checkpoint_path/{run_id}/{checkpoint_id}": { + "/twinkle/checkpoint_path/{run_id}/{checkpoint_id:path}": { "GET": { - "operationId": "get_checkpoint_path_twinkle_checkpoint_path__run_id___checkpoint_id__get", - "parameters": [ + "body": [], + "cookies": [], + "headers": [], + "operationId": "get_checkpoint_path", + "path": [ { - "in": "path", "name": "run_id", "required": true, "schema": { - "title": "Run Id", "type": "string" } }, { - "in": "path", "name": "checkpoint_id", "required": true, "schema": { - "title": "Checkpoint Id", "type": "string" } } ], - "responses": [ - "200", - "422" - ] + "query": [], + "response": { + "description": "Response body for the /checkpoint_path endpoint.", + "properties": { + "path": { + "title": "Path", + "type": "string" + }, + "twinkle_path": { + "title": "Twinkle Path", + "type": "string" + } + }, + "required": [ + "path", + "twinkle_path" + ], + "title": "CheckpointPathResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/create_session": { "POST": { - "operationId": "create_session_twinkle_create_session_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "description": "Request body for POST /twinkle/create_session.", + "properties": { + "metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Metadata" + } + }, + "title": "CreateSessionRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "create_session", + "path": [], + "query": [], + "response": { + "description": "Response body for POST /twinkle/create_session.", + "properties": { + "session_id": { + "title": "Session Id", + "type": "string" + } + }, + "required": [ + "session_id" + ], + "title": "CreateSessionResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/get_server_capabilities": { "GET": { - "operationId": "get_server_capabilities_twinkle_get_server_capabilities_get", - "parameters": [], - "responses": [ - "200" - ] + "body": [], + "cookies": [], + "headers": [], + "operationId": "get_server_capabilities", + "path": [], + "query": [], + "response": { + "$defs": { + "SupportedModel": { + "description": "Information about a supported model.", + "properties": { + "model_name": { + "title": "Model Name", + "type": "string" + } + }, + "required": [ + "model_name" + ], + "title": "SupportedModel", + "type": "object" + } + }, + "description": "Response body for the /get_server_capabilities endpoint.", + "properties": { + "supported_models": { + "items": { + "$ref": "#/$defs/SupportedModel" + }, + "title": "Supported Models", + "type": "array" + } + }, + "required": [ + "supported_models" + ], + "title": "GetServerCapabilitiesResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/healthz": { "GET": { - "operationId": "healthz_twinkle_healthz_get", - "parameters": [], - "responses": [ - "200" - ] + "body": [], + "cookies": [], + "headers": [], + "operationId": "healthz", + "path": [], + "query": [], + "response": { + "properties": { + "status": { + "title": "Status", + "type": "string" + } + }, + "required": [ + "status" + ], + "title": "HealthResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/healthz/deep": { "GET": { - "operationId": "healthz_deep_twinkle_healthz_deep_get", - "parameters": [], - "responses": [ - "200" - ] + "body": [], + "cookies": [], + "headers": [], + "operationId": "healthz_deep", + "path": [], + "query": [], + "response": { + "pythonType": "dict" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/session_heartbeat": { "POST": { - "operationId": "session_heartbeat_twinkle_session_heartbeat_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "description": "Request body for POST /twinkle/session_heartbeat.", + "properties": { + "session_id": { + "title": "Session Id", + "type": "string" + } + }, + "required": [ + "session_id" + ], + "title": "SessionHeartbeatRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "session_heartbeat", + "path": [], + "query": [], + "response": { + "description": "Response body for POST /twinkle/session_heartbeat.", + "properties": {}, + "title": "SessionHeartbeatResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/status": { "GET": { - "operationId": "status_twinkle_status_get", - "parameters": [], - "responses": [ - "200" - ] + "body": [], + "cookies": [], + "headers": [], + "operationId": "status", + "path": [], + "query": [], + "response": { + "pythonType": "dict" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/training_runs": { "GET": { - "operationId": "get_training_runs_twinkle_training_runs_get", - "parameters": [ + "body": [], + "cookies": [], + "headers": [], + "operationId": "get_training_runs", + "path": [], + "query": [ { - "in": "query", "name": "limit", "required": false, "schema": { - "default": 20, - "title": "Limit", "type": "integer" } }, { - "in": "query", "name": "offset", "required": false, "schema": { - "default": 0, - "title": "Offset", "type": "integer" } } ], - "responses": [ - "200", - "422" - ] + "response": { + "$defs": { + "Cursor": { + "properties": { + "limit": { + "title": "Limit", + "type": "integer" + }, + "offset": { + "title": "Offset", + "type": "integer" + }, + "total_count": { + "title": "Total Count", + "type": "integer" + } + }, + "required": [ + "limit", + "offset", + "total_count" + ], + "title": "Cursor", + "type": "object" + }, + "TrainingRun": { + "description": "Twinkle training run model.", + "properties": { + "base_model": { + "title": "Base Model", + "type": "string" + }, + "corrupted": { + "default": false, + "title": "Corrupted", + "type": "boolean" + }, + "is_lora": { + "default": false, + "title": "Is Lora", + "type": "boolean" + }, + "last_checkpoint": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Last Checkpoint" + }, + "last_request_time": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Last Request Time" + }, + "last_sampler_checkpoint": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Last Sampler Checkpoint" + }, + "lora_rank": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lora Rank" + }, + "model_owner": { + "title": "Model Owner", + "type": "string" + }, + "save_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Save Dir" + }, + "training_run_id": { + "title": "Training Run Id", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "training_run_id", + "base_model", + "model_owner" + ], + "title": "TrainingRun", + "type": "object" + } + }, + "properties": { + "cursor": { + "$ref": "#/$defs/Cursor" + }, + "training_runs": { + "items": { + "$ref": "#/$defs/TrainingRun" + }, + "title": "Training Runs", + "type": "array" + } + }, + "required": [ + "training_runs", + "cursor" + ], + "title": "TrainingRunsResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/training_runs/{run_id}": { "GET": { - "operationId": "get_training_run_twinkle_training_runs__run_id__get", - "parameters": [ + "body": [], + "cookies": [], + "headers": [], + "operationId": "get_training_run", + "path": [ { - "in": "path", "name": "run_id", "required": true, "schema": { - "title": "Run Id", "type": "string" } } ], - "responses": [ - "200", - "422" - ] + "query": [], + "response": { + "description": "Twinkle training run model.", + "properties": { + "base_model": { + "title": "Base Model", + "type": "string" + }, + "corrupted": { + "default": false, + "title": "Corrupted", + "type": "boolean" + }, + "is_lora": { + "default": false, + "title": "Is Lora", + "type": "boolean" + }, + "last_checkpoint": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Last Checkpoint" + }, + "last_request_time": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Last Request Time" + }, + "last_sampler_checkpoint": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Last Sampler Checkpoint" + }, + "lora_rank": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lora Rank" + }, + "model_owner": { + "title": "Model Owner", + "type": "string" + }, + "save_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Save Dir" + }, + "training_run_id": { + "title": "Training Run Id", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "training_run_id", + "base_model", + "model_owner" + ], + "title": "TrainingRun", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/training_runs/{run_id}/checkpoints": { "GET": { - "operationId": "get_run_checkpoints_twinkle_training_runs__run_id__checkpoints_get", - "parameters": [ + "body": [], + "cookies": [], + "headers": [], + "operationId": "get_run_checkpoints", + "path": [ { - "in": "path", "name": "run_id", "required": true, "schema": { - "title": "Run Id", "type": "string" } } ], - "responses": [ - "200", - "422" - ] + "query": [], + "response": { + "$defs": { + "Checkpoint": { + "description": "Twinkle checkpoint model.", + "properties": { + "base_model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Model" + }, + "checkpoint_id": { + "title": "Checkpoint Id", + "type": "string" + }, + "checkpoint_type": { + "title": "Checkpoint Type", + "type": "string" + }, + "is_lora": { + "default": false, + "title": "Is Lora", + "type": "boolean" + }, + "lora_rank": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lora Rank" + }, + "public": { + "default": false, + "title": "Public", + "type": "boolean" + }, + "size_bytes": { + "title": "Size Bytes", + "type": "integer" + }, + "time": { + "format": "date-time", + "title": "Time", + "type": "string" + }, + "train_attn": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Train Attn" + }, + "train_mlp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Train Mlp" + }, + "train_unembed": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Train Unembed" + }, + "twinkle_path": { + "title": "Twinkle Path", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "checkpoint_id", + "checkpoint_type", + "time", + "size_bytes", + "twinkle_path" + ], + "title": "Checkpoint", + "type": "object" + }, + "Cursor": { + "properties": { + "limit": { + "title": "Limit", + "type": "integer" + }, + "offset": { + "title": "Offset", + "type": "integer" + }, + "total_count": { + "title": "Total Count", + "type": "integer" + } + }, + "required": [ + "limit", + "offset", + "total_count" + ], + "title": "Cursor", + "type": "object" + } + }, + "properties": { + "checkpoints": { + "items": { + "$ref": "#/$defs/Checkpoint" + }, + "title": "Checkpoints", + "type": "array" + }, + "cursor": { + "anyOf": [ + { + "$ref": "#/$defs/Cursor" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "checkpoints" + ], + "title": "CheckpointsListResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, - "/twinkle/training_runs/{run_id}/checkpoints/{checkpoint_id}": { + "/twinkle/training_runs/{run_id}/checkpoints/{checkpoint_id:path}": { "DELETE": { - "operationId": "delete_run_checkpoint_twinkle_training_runs__run_id__checkpoints__checkpoint_id__delete", - "parameters": [ + "body": [], + "cookies": [], + "headers": [], + "operationId": "delete_run_checkpoint", + "path": [ { - "in": "path", "name": "run_id", "required": true, "schema": { - "title": "Run Id", "type": "string" } }, { - "in": "path", "name": "checkpoint_id", "required": true, "schema": { - "title": "Checkpoint Id", "type": "string" } } ], - "responses": [ - "200", - "422" - ] + "query": [], + "response": { + "properties": { + "message": { + "title": "Message", + "type": "string" + }, + "success": { + "title": "Success", + "type": "boolean" + } + }, + "required": [ + "success", + "message" + ], + "title": "DeleteCheckpointResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/weights_info": { "POST": { - "operationId": "weights_info_twinkle_weights_info_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "properties": { + "twinkle_path": { + "title": "Twinkle Path", + "type": "string" + } + }, + "required": [ + "twinkle_path" + ], + "title": "WeightsInfoRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "weights_info", + "path": [], + "query": [], + "response": { + "description": "Twinkle weights info response.", + "properties": { + "base_model": { + "title": "Base Model", + "type": "string" + }, + "is_lora": { + "default": false, + "title": "Is Lora", + "type": "boolean" + }, + "lora_rank": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lora Rank" + }, + "model_owner": { + "title": "Model Owner", + "type": "string" + }, + "training_run_id": { + "title": "Training Run Id", + "type": "string" + } + }, + "required": [ + "training_run_id", + "base_model", + "model_owner" + ], + "title": "WeightsInfoResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/unload_model": { "POST": { - "operationId": "unload_model_unload_model_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "model_id": { + "title": "Model Id", + "type": "string" + }, + "type": { + "const": "unload_model", + "default": "unload_model", + "title": "Type", + "type": "string" + } + }, + "required": [ + "model_id" + ], + "title": "UnloadModelRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "unload_model", + "path": [], + "query": [], + "response": {}, + "responses": {}, + "statusCode": 200 } }, "/weights_info": { "POST": { - "operationId": "weights_info_weights_info_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": {}, + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "weights_info", + "path": [], + "query": [], + "response": { + "description": "Minimal information for loading public checkpoints.", + "properties": { + "base_model": { + "title": "Base Model", + "type": "string" + }, + "is_lora": { + "title": "Is Lora", + "type": "boolean" + }, + "lora_rank": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lora Rank" + }, + "train_attn": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Train Attn" + }, + "train_mlp": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Train Mlp" + }, + "train_unembed": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Train Unembed" + } + }, + "required": [ + "base_model", + "is_lora" + ], + "title": "WeightsInfoResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } } } @@ -536,421 +3940,2897 @@ "paths": { "/healthz": { "GET": { - "operationId": "model_healthz_healthz_get", - "parameters": [], - "responses": [ - "200" - ] + "body": [], + "cookies": [], + "headers": [], + "operationId": "model_healthz", + "path": [], + "query": [], + "response": { + "pythonType": "dict" + }, + "responses": {}, + "statusCode": 200 } }, "/tinker/create_model": { "POST": { - "operationId": "create_model_tinker_create_model_post", - "parameters": [], - "responses": [ - "200", - "422" - ] - } - }, - "/tinker/forward": { - "POST": { - "operationId": "forward_tinker_forward_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "LoraConfig": { + "additionalProperties": false, + "properties": { + "rank": { + "title": "Rank", + "type": "integer" + }, + "seed": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seed" + }, + "train_attn": { + "default": true, + "title": "Train Attn", + "type": "boolean" + }, + "train_mlp": { + "default": true, + "title": "Train Mlp", + "type": "boolean" + }, + "train_unembed": { + "default": true, + "title": "Train Unembed", + "type": "boolean" + } + }, + "required": [ + "rank" + ], + "title": "LoraConfig", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "base_model": { + "title": "Base Model", + "type": "string" + }, + "lora_config": { + "anyOf": [ + { + "$ref": "#/$defs/LoraConfig" + }, + { + "type": "null" + } + ], + "default": null + }, + "model_seq_id": { + "title": "Model Seq Id", + "type": "integer" + }, + "session_id": { + "title": "Session Id", + "type": "string" + }, + "type": { + "const": "create_model", + "default": "create_model", + "title": "Type", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "session_id", + "model_seq_id", + "base_model" + ], + "title": "CreateModelRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "create_model", + "path": [], + "query": [], + "response": { + "properties": { + "model_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Id" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "sample_sequence_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sample Sequence Ids" + } + }, + "required": [ + "request_id" + ], + "title": "UntypedAPIFuture", + "type": "object" + }, + "responses": {}, + "statusCode": 200 + } + }, + "/tinker/forward": { + "POST": { + "body": [ + { + "name": "body", + "required": true, + "schema": { + "properties": { + "forward_input": { + "properties": { + "data": { + "items": { + "properties": { + "loss_fn_input_spans": { + "pythonType": "'Optional[ProvenanceSpans]'" + }, + "loss_fn_inputs": { + "pythonType": "'LossFnInputs'" + }, + "model_input": { + "pythonType": "'ModelInput'" + }, + "model_input_spans": { + "pythonType": "'Optional[ProvenanceSpans]'" + } + }, + "required": [ + "model_input" + ], + "type": "object" + }, + "type": "array" + }, + "loss_fn": { + "enum": [ + "cross_entropy", + "importance_sampling", + "ppo", + "cispo", + "dro" + ] + }, + "loss_fn_config": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "data", + "loss_fn" + ], + "type": "object" + }, + "model_id": { + "type": "string" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "forward_input", + "model_id" + ], + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "forward", + "path": [], + "query": [], + "response": { + "properties": { + "model_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Id" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "sample_sequence_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sample Sequence Ids" + } + }, + "required": [ + "request_id" + ], + "title": "UntypedAPIFuture", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/tinker/forward_backward": { "POST": { - "operationId": "forward_backward_tinker_forward_backward_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "properties": { + "forward_backward_input": { + "properties": { + "data": { + "items": { + "properties": { + "loss_fn_input_spans": { + "pythonType": "'Optional[ProvenanceSpans]'" + }, + "loss_fn_inputs": { + "pythonType": "'LossFnInputs'" + }, + "model_input": { + "pythonType": "'ModelInput'" + }, + "model_input_spans": { + "pythonType": "'Optional[ProvenanceSpans]'" + } + }, + "required": [ + "model_input" + ], + "type": "object" + }, + "type": "array" + }, + "loss_fn": { + "enum": [ + "cross_entropy", + "importance_sampling", + "ppo", + "cispo", + "dro" + ] + }, + "loss_fn_config": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + } + ] + }, + "type": "object" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "data", + "loss_fn" + ], + "type": "object" + }, + "model_id": { + "type": "string" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "forward_backward_input", + "model_id" + ], + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "forward_backward", + "path": [], + "query": [], + "response": { + "properties": { + "model_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Id" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "sample_sequence_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sample Sequence Ids" + } + }, + "required": [ + "request_id" + ], + "title": "UntypedAPIFuture", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/tinker/get_info": { "POST": { - "operationId": "get_info_tinker_get_info_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "model_id": { + "title": "Model Id", + "type": "string" + }, + "type": { + "const": "get_info", + "default": "get_info", + "title": "Type", + "type": "string" + } + }, + "required": [ + "model_id" + ], + "title": "GetInfoRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "get_info", + "path": [], + "query": [], + "response": { + "$defs": { + "ModelData": { + "description": "Metadata about a model's architecture and configuration.", + "properties": { + "arch": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Arch" + }, + "model_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Name" + }, + "tokenizer_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Tokenizer Id" + } + }, + "title": "ModelData", + "type": "object" + } + }, + "description": "Response containing information about a training client's model.", + "properties": { + "is_lora": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Is Lora" + }, + "lora_rank": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Lora Rank" + }, + "model_data": { + "$ref": "#/$defs/ModelData" + }, + "model_id": { + "title": "Model Id", + "type": "string" + }, + "model_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Name" + }, + "type": { + "anyOf": [ + { + "const": "get_info", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Type" + } + }, + "required": [ + "model_data", + "model_id" + ], + "title": "GetInfoResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/tinker/load_weights": { "POST": { - "operationId": "load_weights_tinker_load_weights_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "base_model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Model" + }, + "model_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Id" + }, + "model_seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Seq Id" + }, + "optimizer": { + "title": "Optimizer", + "type": "boolean" + }, + "path": { + "title": "Path", + "type": "string" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seq Id" + }, + "session_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Session Id" + }, + "type": { + "const": "load_weights", + "default": "load_weights", + "title": "Type", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + }, + "weights_access_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Weights Access Token" + } + }, + "required": [ + "path", + "optimizer" + ], + "title": "LoadWeightsRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "load_weights", + "path": [], + "query": [], + "response": { + "properties": { + "model_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Id" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "sample_sequence_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sample Sequence Ids" + } + }, + "required": [ + "request_id" + ], + "title": "UntypedAPIFuture", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/tinker/optim_step": { "POST": { - "operationId": "optim_step_tinker_optim_step_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "AdamParams": { + "additionalProperties": false, + "properties": { + "beta1": { + "default": 0.9, + "title": "Beta1", + "type": "number" + }, + "beta2": { + "default": 0.95, + "title": "Beta2", + "type": "number" + }, + "eps": { + "default": 1e-12, + "title": "Eps", + "type": "number" + }, + "grad_clip_norm": { + "default": 0.0, + "title": "Grad Clip Norm", + "type": "number" + }, + "learning_rate": { + "default": 0.0001, + "title": "Learning Rate", + "type": "number" + }, + "weight_decay": { + "default": 0.0, + "title": "Weight Decay", + "type": "number" + } + }, + "title": "AdamParams", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "adam_params": { + "$ref": "#/$defs/AdamParams" + }, + "model_id": { + "title": "Model Id", + "type": "string" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seq Id" + }, + "type": { + "const": "optim_step", + "default": "optim_step", + "title": "Type", + "type": "string" + } + }, + "required": [ + "adam_params", + "model_id" + ], + "title": "OptimStepRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "optim_step", + "path": [], + "query": [], + "response": { + "properties": { + "model_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Id" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "sample_sequence_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sample Sequence Ids" + } + }, + "required": [ + "request_id" + ], + "title": "UntypedAPIFuture", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/tinker/save_weights": { "POST": { - "operationId": "save_weights_tinker_save_weights_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "model_id": { + "title": "Model Id", + "type": "string" + }, + "overwrite": { + "default": false, + "title": "Overwrite", + "type": "boolean" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seq Id" + }, + "ttl_seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Ttl Seconds" + }, + "type": { + "const": "save_weights", + "default": "save_weights", + "title": "Type", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "model_id" + ], + "title": "SaveWeightsRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "save_weights", + "path": [], + "query": [], + "response": { + "properties": { + "model_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Id" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "sample_sequence_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sample Sequence Ids" + } + }, + "required": [ + "request_id" + ], + "title": "UntypedAPIFuture", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/tinker/save_weights_for_sampler": { "POST": { - "operationId": "save_weights_for_sampler_tinker_save_weights_for_sampler_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "model_id": { + "title": "Model Id", + "type": "string" + }, + "path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "sampling_session_seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sampling Session Seq Id" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seq Id" + }, + "ttl_seconds": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Ttl Seconds" + }, + "type": { + "const": "save_weights_for_sampler", + "default": "save_weights_for_sampler", + "title": "Type", + "type": "string" + }, + "user_metadata": { + "anyOf": [ + { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "User Metadata" + } + }, + "required": [ + "model_id" + ], + "title": "SaveWeightsForSamplerRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "save_weights_for_sampler", + "path": [], + "query": [], + "response": { + "properties": { + "model_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Id" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "sample_sequence_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sample Sequence Ids" + } + }, + "required": [ + "request_id" + ], + "title": "UntypedAPIFuture", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/tinker/unload_model": { "POST": { - "operationId": "unload_model_tinker_unload_model_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": false, + "properties": { + "model_id": { + "title": "Model Id", + "type": "string" + }, + "type": { + "const": "unload_model", + "default": "unload_model", + "title": "Type", + "type": "string" + } + }, + "required": [ + "model_id" + ], + "title": "UnloadModelRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "unload_model", + "path": [], + "query": [], + "response": { + "properties": { + "model_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Id" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "sample_sequence_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sample Sequence Ids" + } + }, + "required": [ + "request_id" + ], + "title": "UntypedAPIFuture", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/add_adapter_to_model": { "POST": { - "operationId": "add_adapter_to_model_twinkle_add_adapter_to_model_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "config": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Config" + }, + "save_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Save Dir" + } + }, + "required": [ + "adapter_name" + ], + "title": "AddAdapterRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "add_adapter_to_model", + "path": [], + "query": [], + "response": { + "description": "Response body for the /add_adapter_to_sampler endpoint.", + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "status": { + "default": "ok", + "title": "Status", + "type": "string" + } + }, + "required": [ + "adapter_name" + ], + "title": "AddAdapterResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/add_metric": { "POST": { - "operationId": "add_metric_twinkle_add_metric_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "is_training": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Is Training" + }, + "metric_cls": { + "title": "Metric Cls", + "type": "string" + } + }, + "required": [ + "metric_cls", + "adapter_name" + ], + "title": "AddMetricRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "add_metric", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/apply_patch": { "POST": { - "operationId": "apply_patch_twinkle_apply_patch_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "patch_cls": { + "title": "Patch Cls", + "type": "string" + } + }, + "required": [ + "patch_cls", + "adapter_name" + ], + "title": "ApplyPatchRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "apply_patch", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/backward": { "POST": { - "operationId": "backward_twinkle_backward_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + } + }, + "required": [ + "adapter_name" + ], + "title": "AdapterRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "backward", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/calculate_loss": { "POST": { - "operationId": "calculate_loss_twinkle_calculate_loss_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + } + }, + "required": [ + "adapter_name" + ], + "title": "AdapterRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "calculate_loss", + "path": [], + "query": [], + "response": { + "description": "Response for /calculate_loss endpoint (returns float).", + "properties": { + "result": { + "title": "Result", + "type": "number" + } + }, + "required": [ + "result" + ], + "title": "CalculateLossResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/calculate_metric": { "POST": { - "operationId": "calculate_metric_twinkle_calculate_metric_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "is_training": { + "default": true, + "title": "Is Training", + "type": "boolean" + } + }, + "required": [ + "adapter_name" + ], + "title": "CalculateMetricRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "calculate_metric", + "path": [], + "query": [], + "response": { + "description": "Response for /calculate_metric endpoint (returns Dict).", + "properties": { + "result": { + "additionalProperties": true, + "title": "Result", + "type": "object" + } + }, + "required": [ + "result" + ], + "title": "CalculateMetricResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/clip_grad_and_step": { "POST": { - "operationId": "clip_grad_and_step_twinkle_clip_grad_and_step_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "max_grad_norm": { + "default": 1.0, + "title": "Max Grad Norm", + "type": "number" + }, + "norm_type": { + "default": 2, + "title": "Norm Type", + "type": "integer" + } + }, + "required": [ + "adapter_name" + ], + "title": "ClipGradAndStepRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "clip_grad_and_step", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/clip_grad_norm": { "POST": { - "operationId": "clip_grad_norm_twinkle_clip_grad_norm_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + } + }, + "required": [ + "adapter_name" + ], + "title": "AdapterRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "clip_grad_norm", + "path": [], + "query": [], + "response": { + "description": "Response for /clip_grad_norm endpoint (returns float as str).", + "properties": { + "result": { + "title": "Result", + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "ClipGradNormResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/create": { "POST": { - "operationId": "create_twinkle_create_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": {}, + "title": "CreateRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "create", + "path": [], + "query": [], + "response": { + "description": "Response for /create endpoint.", + "properties": { + "status": { + "default": "ok", + "title": "Status", + "type": "string" + } + }, + "title": "CreateResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/forward": { "POST": { - "operationId": "forward_twinkle_forward_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "inputs": { + "title": "Inputs" + } + }, + "required": [ + "inputs", + "adapter_name" + ], + "title": "ForwardRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "forward", + "path": [], + "query": [], + "response": { + "description": "Response for /forward and /forward_only endpoints (returns ModelOutput).", + "properties": { + "result": { + "title": "Result" + } + }, + "required": [ + "result" + ], + "title": "ForwardResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/forward_backward": { "POST": { - "operationId": "forward_backward_twinkle_forward_backward_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "inputs": { + "title": "Inputs" + } + }, + "required": [ + "inputs", + "adapter_name" + ], + "title": "ForwardRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "forward_backward", + "path": [], + "query": [], + "response": { + "description": "Response for /forward_backward endpoint (returns ModelOutput).", + "properties": { + "result": { + "title": "Result" + } + }, + "required": [ + "result" + ], + "title": "ForwardBackwardResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/forward_backward_from_data_plane": { "POST": { - "operationId": "forward_backward_from_data_plane_twinkle_forward_backward_from_data_plane_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "DataRef": { + "description": "Opaque reference to rows stored in the server-side TransferQueue.", + "properties": { + "fields": { + "items": { + "type": "string" + }, + "title": "Fields", + "type": "array" + }, + "kind": { + "default": "data", + "title": "Kind", + "type": "string" + }, + "num_tokens": { + "default": 0, + "title": "Num Tokens", + "type": "integer" + }, + "ref_id": { + "title": "Ref Id", + "type": "string" + }, + "size": { + "title": "Size", + "type": "integer" + } + }, + "required": [ + "ref_id", + "size" + ], + "title": "DataRef", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "input_field": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Field" + }, + "input_refs": { + "items": { + "$ref": "#/$defs/DataRef" + }, + "minItems": 1, + "title": "Input Refs", + "type": "array" + }, + "kwarg_fields": { + "additionalProperties": { + "type": "string" + }, + "title": "Kwarg Fields", + "type": "object" + } + }, + "required": [ + "input_refs", + "adapter_name" + ], + "title": "DataPlaneForwardRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "forward_backward_from_data_plane", + "path": [], + "query": [], + "response": { + "description": "Response for /forward_backward endpoint (returns ModelOutput).", + "properties": { + "result": { + "title": "Result" + } + }, + "required": [ + "result" + ], + "title": "ForwardBackwardResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/forward_from_data_plane": { "POST": { - "operationId": "forward_from_data_plane_twinkle_forward_from_data_plane_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "DataRef": { + "description": "Opaque reference to rows stored in the server-side TransferQueue.", + "properties": { + "fields": { + "items": { + "type": "string" + }, + "title": "Fields", + "type": "array" + }, + "kind": { + "default": "data", + "title": "Kind", + "type": "string" + }, + "num_tokens": { + "default": 0, + "title": "Num Tokens", + "type": "integer" + }, + "ref_id": { + "title": "Ref Id", + "type": "string" + }, + "size": { + "title": "Size", + "type": "integer" + } + }, + "required": [ + "ref_id", + "size" + ], + "title": "DataRef", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "input_field": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Field" + }, + "input_refs": { + "items": { + "$ref": "#/$defs/DataRef" + }, + "minItems": 1, + "title": "Input Refs", + "type": "array" + }, + "kwarg_fields": { + "additionalProperties": { + "type": "string" + }, + "title": "Kwarg Fields", + "type": "object" + } + }, + "required": [ + "input_refs", + "adapter_name" + ], + "title": "DataPlaneForwardRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "forward_from_data_plane", + "path": [], + "query": [], + "response": { + "description": "Response for /forward and /forward_only endpoints (returns ModelOutput).", + "properties": { + "result": { + "title": "Result" + } + }, + "required": [ + "result" + ], + "title": "ForwardResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/forward_only": { "POST": { - "operationId": "forward_only_twinkle_forward_only_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Adapter Name" + }, + "inputs": { + "title": "Inputs" + } + }, + "required": [ + "inputs" + ], + "title": "ForwardOnlyRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "forward_only", + "path": [], + "query": [], + "response": { + "description": "Response for /forward and /forward_only endpoints (returns ModelOutput).", + "properties": { + "result": { + "title": "Result" + } + }, + "required": [ + "result" + ], + "title": "ForwardResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/forward_only_from_data_plane": { "POST": { - "operationId": "forward_only_from_data_plane_twinkle_forward_only_from_data_plane_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "DataRef": { + "description": "Opaque reference to rows stored in the server-side TransferQueue.", + "properties": { + "fields": { + "items": { + "type": "string" + }, + "title": "Fields", + "type": "array" + }, + "kind": { + "default": "data", + "title": "Kind", + "type": "string" + }, + "num_tokens": { + "default": 0, + "title": "Num Tokens", + "type": "integer" + }, + "ref_id": { + "title": "Ref Id", + "type": "string" + }, + "size": { + "title": "Size", + "type": "integer" + } + }, + "required": [ + "ref_id", + "size" + ], + "title": "DataRef", + "type": "object" + } + }, + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "input_field": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Input Field" + }, + "input_refs": { + "items": { + "$ref": "#/$defs/DataRef" + }, + "minItems": 1, + "title": "Input Refs", + "type": "array" + }, + "kwarg_fields": { + "additionalProperties": { + "type": "string" + }, + "title": "Kwarg Fields", + "type": "object" + }, + "output_fields": { + "additionalProperties": { + "type": "string" + }, + "title": "Output Fields", + "type": "object" + }, + "output_ref": { + "anyOf": [ + { + "$ref": "#/$defs/DataRef" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "required": [ + "input_refs", + "adapter_name" + ], + "title": "DataPlaneForwardOnlyRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "forward_only_from_data_plane", + "path": [], + "query": [], + "response": { + "description": "Response for /forward and /forward_only endpoints (returns ModelOutput).", + "properties": { + "result": { + "title": "Result" + } + }, + "required": [ + "result" + ], + "title": "ForwardResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/get_state_dict": { "POST": { - "operationId": "get_state_dict_twinkle_get_state_dict_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + } + }, + "required": [ + "adapter_name" + ], + "title": "GetStateDictRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "get_state_dict", + "path": [], + "query": [], + "response": { + "description": "Response for /get_state_dict endpoint (returns Dict).", + "properties": { + "result": { + "additionalProperties": true, + "title": "Result", + "type": "object" + } + }, + "required": [ + "result" + ], + "title": "GetStateDictResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/get_train_configs": { "POST": { - "operationId": "get_train_configs_twinkle_get_train_configs_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + } + }, + "required": [ + "adapter_name" + ], + "title": "AdapterRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "get_train_configs", + "path": [], + "query": [], + "response": { + "description": "Response for /get_train_configs endpoint (returns str).", + "properties": { + "result": { + "title": "Result", + "type": "string" + } + }, + "required": [ + "result" + ], + "title": "GetTrainConfigsResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/load": { "POST": { - "operationId": "load_twinkle_load_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "load_optimizer": { + "default": false, + "title": "Load Optimizer", + "type": "boolean" + }, + "name": { + "title": "Name", + "type": "string" + } + }, + "required": [ + "adapter_name", + "name" + ], + "title": "LoadRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "load", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/lr_step": { "POST": { - "operationId": "lr_step_twinkle_lr_step_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + } + }, + "required": [ + "adapter_name" + ], + "title": "AdapterRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "lr_step", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/remove_adapter": { "POST": { - "operationId": "remove_adapter_twinkle_remove_adapter_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + } + }, + "required": [ + "adapter_name" + ], + "title": "AdapterRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "remove_adapter", + "path": [], + "query": [], + "response": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/resume_from_checkpoint": { "POST": { - "operationId": "resume_from_checkpoint_twinkle_resume_from_checkpoint_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "description": "Request for /resume_from_checkpoint endpoint.", + "properties": { + "adapter_name": { + "default": "", + "title": "Adapter Name", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + }, + "resume_only_model": { + "default": false, + "title": "Resume Only Model", + "type": "boolean" + } + }, + "required": [ + "name" + ], + "title": "ResumeFromCheckpointRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "resume_from_checkpoint", + "path": [], + "query": [], + "response": { + "description": "Response for /resume_from_checkpoint endpoint.", + "properties": { + "result": { + "additionalProperties": true, + "title": "Result", + "type": "object" + } + }, + "required": [ + "result" + ], + "title": "TrainingProgressResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/save": { "POST": { - "operationId": "save_twinkle_save_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "is_sampler": { + "default": false, + "title": "Is Sampler", + "type": "boolean" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Name" + }, + "save_optimizer": { + "default": false, + "title": "Save Optimizer", + "type": "boolean" + } + }, + "required": [ + "adapter_name" + ], + "title": "SaveRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "save", + "path": [], + "query": [], + "response": { + "description": "Response for /save endpoint (returns twinkle path + checkpoint dir).", + "properties": { + "checkpoint_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Checkpoint Dir" + }, + "twinkle_path": { + "title": "Twinkle Path", + "type": "string" + } + }, + "required": [ + "twinkle_path" + ], + "title": "SaveResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/set_loss": { "POST": { - "operationId": "set_loss_twinkle_set_loss_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "loss_cls": { + "title": "Loss Cls", + "type": "string" + } + }, + "required": [ + "loss_cls", + "adapter_name" + ], + "title": "SetLossRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "set_loss", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/set_lr_scheduler": { "POST": { - "operationId": "set_lr_scheduler_twinkle_set_lr_scheduler_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "scheduler_cls": { + "title": "Scheduler Cls", + "type": "string" + } + }, + "required": [ + "scheduler_cls", + "adapter_name" + ], + "title": "SetLrSchedulerRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "set_lr_scheduler", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/set_optimizer": { "POST": { - "operationId": "set_optimizer_twinkle_set_optimizer_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "optimizer_cls": { + "title": "Optimizer Cls", + "type": "string" + } + }, + "required": [ + "optimizer_cls", + "adapter_name" + ], + "title": "SetOptimizerRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "set_optimizer", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/set_processor": { "POST": { - "operationId": "set_processor_twinkle_set_processor_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "processor_cls": { + "title": "Processor Cls", + "type": "string" + } + }, + "required": [ + "processor_cls", + "adapter_name" + ], + "title": "SetProcessorRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "set_processor", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/set_template": { "POST": { - "operationId": "set_template_twinkle_set_template_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "template_cls": { + "title": "Template Cls", + "type": "string" + } + }, + "required": [ + "template_cls", + "adapter_name" + ], + "title": "SetTemplateRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "set_template", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/step": { "POST": { - "operationId": "step_twinkle_step_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + } + }, + "required": [ + "adapter_name" + ], + "title": "AdapterRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "step", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/upload_status/{request_id}": { "GET": { - "operationId": "upload_status_twinkle_upload_status__request_id__get", - "parameters": [ + "body": [], + "cookies": [], + "headers": [], + "operationId": "upload_status", + "path": [ { - "in": "path", "name": "request_id", "required": true, "schema": { - "title": "Request Id", "type": "string" } } ], - "responses": [ - "200", - "422" - ] + "query": [], + "response": { + "description": "Response for /upload_status/{request_id} endpoint.", + "properties": { + "error": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Error" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "status": { + "title": "Status", + "type": "string" + } + }, + "required": [ + "request_id", + "status" + ], + "title": "UploadStatusResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/upload_to_hub": { "POST": { - "operationId": "upload_to_hub_twinkle_upload_to_hub_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "async_upload": { + "default": false, + "title": "Async Upload", + "type": "boolean" + }, + "checkpoint_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + } + ], + "title": "Checkpoint Dir" + }, + "hub_model_id": { + "title": "Hub Model Id", + "type": "string" + }, + "hub_token": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Hub Token" + } + }, + "required": [ + "checkpoint_dir", + "hub_model_id" + ], + "title": "UploadToHubRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "upload_to_hub", + "path": [], + "query": [], + "response": { + "description": "Response for /upload_to_hub endpoint.", + "properties": { + "request_id": { + "title": "Request Id", + "type": "string" + } + }, + "required": [ + "request_id" + ], + "title": "UploadToHubResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/zero_grad": { "POST": { - "operationId": "zero_grad_twinkle_zero_grad_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + } + }, + "required": [ + "adapter_name" + ], + "title": "AdapterRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "zero_grad", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } } } @@ -959,22 +6839,101 @@ "paths": { "/twinkle/call": { "POST": { - "operationId": "call_twinkle_call_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "function": { + "title": "Function", + "type": "string" + }, + "processor_id": { + "title": "Processor Id", + "type": "string" + } + }, + "required": [ + "processor_id", + "function" + ], + "title": "ProcessorCallRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "call", + "path": [], + "query": [], + "response": { + "description": "Response body for the /call endpoint.", + "properties": { + "result": { + "title": "Result" + } + }, + "required": [ + "result" + ], + "title": "ProcessorCallResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/create": { "POST": { - "operationId": "create_twinkle_create_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "class_type": { + "title": "Class Type", + "type": "string" + }, + "processor_type": { + "title": "Processor Type", + "type": "string" + } + }, + "required": [ + "processor_type", + "class_type" + ], + "title": "ProcessorCreateRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "create", + "path": [], + "query": [], + "response": { + "description": "Response body for the /create endpoint.", + "properties": { + "processor_id": { + "title": "Processor Id", + "type": "string" + } + }, + "required": [ + "processor_id" + ], + "title": "ProcessorCreateResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } } } @@ -983,91 +6942,1130 @@ "paths": { "/tinker/asample": { "POST": { - "operationId": "asample_tinker_asample_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "DmelChunk": { + "additionalProperties": false, + "properties": { + "dmel": { + "format": "binary", + "title": "Dmel", + "type": "string" + }, + "type": { + "const": "dmel", + "default": "dmel", + "title": "Type", + "type": "string" + } + }, + "required": [ + "dmel" + ], + "title": "DmelChunk", + "type": "object" + }, + "EncodedTextChunk": { + "additionalProperties": false, + "properties": { + "tokens": { + "items": { + "type": "integer" + }, + "title": "Tokens", + "type": "array" + }, + "type": { + "const": "encoded_text", + "default": "encoded_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "tokens" + ], + "title": "EncodedTextChunk", + "type": "object" + }, + "ImageAssetPointerChunk": { + "additionalProperties": false, + "properties": { + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" + }, + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "location": { + "title": "Location", + "type": "string" + }, + "type": { + "const": "image_asset_pointer", + "default": "image_asset_pointer", + "title": "Type", + "type": "string" + } + }, + "required": [ + "format", + "location" + ], + "title": "ImageAssetPointerChunk", + "type": "object" + }, + "ImageChunk": { + "additionalProperties": false, + "properties": { + "data": { + "format": "binary", + "title": "Data", + "type": "string" + }, + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" + }, + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "type": { + "const": "image", + "default": "image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "data", + "format" + ], + "title": "ImageChunk", + "type": "object" + }, + "ModelInput": { + "additionalProperties": false, + "properties": { + "chunks": { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/EncodedTextChunk" + }, + { + "$ref": "#/$defs/ImageAssetPointerChunk" + }, + { + "$ref": "#/$defs/ImageChunk" + }, + { + "$ref": "#/$defs/DmelChunk" + } + ] + }, + "title": "Chunks", + "type": "array" + } + }, + "required": [ + "chunks" + ], + "title": "ModelInput", + "type": "object" + }, + "SamplingParams": { + "properties": { + "max_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Max Tokens" + }, + "seed": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seed" + }, + "stop": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Stop" + }, + "temperature": { + "default": 1, + "title": "Temperature", + "type": "number" + }, + "top_k": { + "default": -1, + "title": "Top K", + "type": "integer" + }, + "top_p": { + "default": 1, + "title": "Top P", + "type": "number" + } + }, + "title": "SamplingParams", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "base_model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Base Model" + }, + "model_path": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Path" + }, + "num_samples": { + "default": 1, + "title": "Num Samples", + "type": "integer" + }, + "prompt": { + "$ref": "#/$defs/ModelInput" + }, + "prompt_logprobs": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Prompt Logprobs" + }, + "record_stability_info": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Record Stability Info" + }, + "sampling_params": { + "$ref": "#/$defs/SamplingParams" + }, + "sampling_session_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sampling Session Id" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seq Id" + }, + "topk_prompt_logprobs": { + "default": 0, + "title": "Topk Prompt Logprobs", + "type": "integer" + }, + "topk_sample_logprobs": { + "default": 0, + "title": "Topk Sample Logprobs", + "type": "integer" + }, + "type": { + "const": "sample", + "default": "sample", + "title": "Type", + "type": "string" + } + }, + "required": [ + "prompt", + "sampling_params" + ], + "title": "SampleRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "asample", + "path": [], + "query": [], + "response": { + "properties": { + "model_id": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Model Id" + }, + "request_id": { + "title": "Request Id", + "type": "string" + }, + "sample_sequence_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sample Sequence Ids" + } + }, + "required": [ + "request_id" + ], + "title": "UntypedAPIFuture", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/add_adapter_to_sampler": { "POST": { - "operationId": "add_adapter_to_sampler_twinkle_add_adapter_to_sampler_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "config": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Config" + }, + "save_dir": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Save Dir" + } + }, + "required": [ + "adapter_name" + ], + "title": "AddAdapterRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "add_adapter_to_sampler", + "path": [], + "query": [], + "response": { + "description": "Response body for the /add_adapter_to_sampler endpoint.", + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "status": { + "default": "ok", + "title": "Status", + "type": "string" + } + }, + "required": [ + "adapter_name" + ], + "title": "AddAdapterResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/apply_patch": { "POST": { - "operationId": "apply_patch_twinkle_apply_patch_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "patch_cls": { + "title": "Patch Cls", + "type": "string" + } + }, + "required": [ + "patch_cls", + "adapter_name" + ], + "title": "ApplyPatchRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "apply_patch", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/create": { "POST": { - "operationId": "create_twinkle_create_post", - "parameters": [], - "responses": [ - "200" - ] + "body": [], + "cookies": [], + "headers": [], + "operationId": "create", + "path": [], + "query": [], + "response": { + "description": "Response for /create endpoint.", + "properties": { + "status": { + "default": "ok", + "title": "Status", + "type": "string" + } + }, + "title": "CreateResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/sample": { "POST": { - "operationId": "sample_twinkle_sample_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "description": "Request body for the /sample endpoint.", + "properties": { + "adapter_name": { + "default": "", + "description": "Adapter name for LoRA inference", + "title": "Adapter Name", + "type": "string" + }, + "adapter_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Adapter URI (twinkle:// path or local path) for LoRA inference", + "title": "Adapter Uri" + }, + "inputs": { + "description": "List of Trajectory or InputFeature dicts", + "title": "Inputs" + }, + "sampling_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Sampling parameters (max_tokens, temperature, num_samples, etc.)", + "title": "Sampling Params" + } + }, + "required": [ + "inputs" + ], + "title": "SampleRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "sample", + "path": [], + "query": [], + "response": { + "$defs": { + "SampleResponseModel": { + "description": "Mirroring twinkle.data_format.SampleResponse.", + "properties": { + "prompt_logprobs": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Prompt Logprobs" + }, + "prompt_token_ids": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Token IDs of the prompt the sequences continue", + "title": "Prompt Token Ids" + }, + "sequences": { + "description": "List of sampled sequences", + "items": { + "$ref": "#/$defs/SampledSequenceModel" + }, + "title": "Sequences", + "type": "array" + }, + "topk_prompt_logprobs": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "items": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "number" + } + ], + "type": "array" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Topk Prompt Logprobs" + } + }, + "required": [ + "sequences" + ], + "title": "SampleResponseModel", + "type": "object" + }, + "SampledSequenceModel": { + "description": "A single sampled sequence, mirroring twinkle.data_format.SampledSequence.", + "properties": { + "decoded": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Decoded text of the sampled sequence", + "title": "Decoded" + }, + "logprobs": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "items": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "integer" + }, + { + "type": "number" + } + ], + "type": "array" + }, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Per-token log-probabilities", + "title": "Logprobs" + }, + "new_input_feature": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Updated InputFeature after sampling (input_ids, labels, etc.)", + "title": "New Input Feature" + }, + "stop_reason": { + "description": "Stop reason: 'length' or 'stop'", + "enum": [ + "length", + "stop", + "abort", + "error" + ], + "title": "Stop Reason", + "type": "string" + }, + "tokens": { + "description": "Token IDs of the sampled sequence", + "items": { + "type": "integer" + }, + "title": "Tokens", + "type": "array" + } + }, + "required": [ + "stop_reason", + "tokens" + ], + "title": "SampledSequenceModel", + "type": "object" + } + }, + "description": "Response body for the /sample endpoint", + "properties": { + "samples": { + "description": "List of sample responses", + "items": { + "$ref": "#/$defs/SampleResponseModel" + }, + "title": "Samples", + "type": "array" + } + }, + "required": [ + "samples" + ], + "title": "SampleResponseModelList", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/sample_stream": { "POST": { - "operationId": "sample_stream_twinkle_sample_stream_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "description": "Request body for the /sample endpoint.", + "properties": { + "adapter_name": { + "default": "", + "description": "Adapter name for LoRA inference", + "title": "Adapter Name", + "type": "string" + }, + "adapter_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Adapter URI (twinkle:// path or local path) for LoRA inference", + "title": "Adapter Uri" + }, + "inputs": { + "description": "List of Trajectory or InputFeature dicts", + "title": "Inputs" + }, + "sampling_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Sampling parameters (max_tokens, temperature, num_samples, etc.)", + "title": "Sampling Params" + } + }, + "required": [ + "inputs" + ], + "title": "SampleRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "sample_stream", + "path": [], + "query": [], + "response": { + "type": "null" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/sample_to_data_plane": { "POST": { - "operationId": "sample_to_data_plane_twinkle_sample_to_data_plane_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "$defs": { + "DataRef": { + "description": "Opaque reference to rows stored in the server-side TransferQueue.", + "properties": { + "fields": { + "items": { + "type": "string" + }, + "title": "Fields", + "type": "array" + }, + "kind": { + "default": "data", + "title": "Kind", + "type": "string" + }, + "num_tokens": { + "default": 0, + "title": "Num Tokens", + "type": "integer" + }, + "ref_id": { + "title": "Ref Id", + "type": "string" + }, + "size": { + "title": "Size", + "type": "integer" + } + }, + "required": [ + "ref_id", + "size" + ], + "title": "DataRef", + "type": "object" + } + }, + "properties": { + "adapter_name": { + "default": "", + "title": "Adapter Name", + "type": "string" + }, + "adapter_uri": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Adapter Uri" + }, + "group_ids": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Group Ids" + }, + "input_ref": { + "anyOf": [ + { + "$ref": "#/$defs/DataRef" + }, + { + "type": "null" + } + ], + "default": null + }, + "inputs": { + "default": null, + "title": "Inputs" + }, + "num_samples": { + "default": 1, + "title": "Num Samples", + "type": "integer" + }, + "policy_version": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Policy Version" + }, + "sampling_params": { + "anyOf": [ + { + "additionalProperties": true, + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Sampling Params" + } + }, + "title": "DataPlaneSampleRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "sample_to_data_plane", + "path": [], + "query": [], + "response": { + "description": "Opaque reference to rows stored in the server-side TransferQueue.", + "properties": { + "fields": { + "items": { + "type": "string" + }, + "title": "Fields", + "type": "array" + }, + "kind": { + "default": "data", + "title": "Kind", + "type": "string" + }, + "num_tokens": { + "default": 0, + "title": "Num Tokens", + "type": "integer" + }, + "ref_id": { + "title": "Ref Id", + "type": "string" + }, + "size": { + "title": "Size", + "type": "integer" + } + }, + "required": [ + "ref_id", + "size" + ], + "title": "DataRef", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/set_template": { "POST": { - "operationId": "set_template_twinkle_set_template_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "additionalProperties": true, + "properties": { + "adapter_name": { + "title": "Adapter Name", + "type": "string" + }, + "template_cls": { + "title": "Template Cls", + "type": "string" + } + }, + "required": [ + "template_cls", + "adapter_name" + ], + "title": "SetTemplateRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "set_template", + "path": [], + "query": [], + "response": { + "description": "Response for /set_template endpoint.", + "properties": { + "status": { + "default": "ok", + "title": "Status", + "type": "string" + } + }, + "title": "SetTemplateResponse", + "type": "object" + }, + "responses": {}, + "statusCode": 200 } }, "/twinkle/unload_adapter_paths": { "POST": { - "operationId": "unload_adapter_paths_twinkle_unload_adapter_paths_post", - "parameters": [], - "responses": [ - "200", - "422" - ] + "body": [ + { + "name": "body", + "required": true, + "schema": { + "properties": { + "adapter_paths": { + "items": { + "type": "string" + }, + "title": "Adapter Paths", + "type": "array" + } + }, + "required": [ + "adapter_paths" + ], + "title": "UnloadAdapterPathsRequest", + "type": "object" + } + } + ], + "cookies": [], + "headers": [], + "operationId": "unload_adapter_paths", + "path": [], + "query": [], + "response": { + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "responses": {}, + "statusCode": 200 } } } diff --git a/tests/server/contract/client_api_harness.py b/tests/server/contract/client_api_harness.py index 0496a1b0..919639bb 100644 --- a/tests/server/contract/client_api_harness.py +++ b/tests/server/contract/client_api_harness.py @@ -2,10 +2,10 @@ """ Client-API contract harness. -Builds the four FastAPI apps used by the Ray Serve deployments (Gateway, Model, -Sampler, Processor) by registering their route-registration helpers against a -fresh FastAPI instance, then extracts the client-facing surface (route paths, -HTTP methods, and request/response schemas) as a stable JSON dict. +Builds the five FastAPI apps used by the Ray Serve deployments (Data Plane, +Gateway, Model, Sampler, Processor) by registering their route-registration helpers against a +fresh FastAPI instance, then extracts route paths, methods, parameters, and +recursive request/response type shapes as a stable JSON dict. Used to: - snapshot the current surface into ``client_api_baseline.json`` before the @@ -25,12 +25,17 @@ """ from __future__ import annotations +import dataclasses import json -from collections.abc import Callable +import sys +import types as pytypes +from collections.abc import Callable, Mapping, Sequence +from enum import Enum from fastapi import FastAPI -from fastapi.openapi.utils import get_openapi +from fastapi.routing import APIRoute from pathlib import Path -from typing import Any +from pydantic import BaseModel +from typing import Annotated, Any, Literal, Union, get_args, get_origin, get_type_hints # ----- App build helpers --------------------------------------------------- # @@ -39,6 +44,14 @@ def _noop_self() -> None: return None +def build_data_plane_app() -> FastAPI: + from twinkle.server.data_plane.handlers import register_data_plane_routes + + app = FastAPI() + register_data_plane_routes(app, _noop_self) + return app + + def build_gateway_app() -> FastAPI: from twinkle.server.gateway.openai_handlers import _register_openai_routes from twinkle.server.gateway.tinker_handlers import _register_tinker_routes @@ -80,6 +93,7 @@ def build_processor_app() -> FastAPI: APP_BUILDERS: dict[str, Callable[[], FastAPI]] = { + 'data_plane': build_data_plane_app, 'gateway': build_gateway_app, 'model': build_model_app, 'sampler': build_sampler_app, @@ -91,41 +105,95 @@ def build_processor_app() -> FastAPI: _HTTP_METHODS = {'GET', 'POST', 'PUT', 'PATCH', 'DELETE'} -def _extract_app_surface(app: FastAPI) -> dict[str, Any]: - """Return a SLIM client-contract view of ``app``'s OpenAPI surface. - - Snapshots, per path and HTTP method, only the stable client-facing contract: - the ``operationId``, the ``parameters``, and the set of response status - codes. The full ``components.schemas`` body and per-operation ``requestBody`` - schema are intentionally NOT snapshotted — they churn on Pydantic / FastAPI - version bumps without representing a real client-contract change. Route - paths, HTTP methods, and response status codes remain frozen. - """ - spec = get_openapi( - title='contract', - version='0.0.0', - routes=app.routes, - ) +def _type_contract(annotation: Any, seen: frozenset[str] = frozenset()) -> Any: + """Build a stable field-level schema for Pydantic models and SDK dataclasses.""" + if annotation is None or annotation is type(None): + return {'type': 'null'} + if annotation is Any: + return {} + origin = get_origin(annotation) + args = get_args(annotation) + if origin is Annotated: + return _type_contract(args[0], seen) + if origin in (Union, pytypes.UnionType): + return {'anyOf': [_type_contract(arg, seen) for arg in args]} + if origin in (list, set, tuple, Sequence): + return {'type': 'array', 'items': _type_contract(args[0], seen) if args else {}} + if origin in (dict, Mapping): + return {'type': 'object', 'additionalProperties': _type_contract(args[1], seen) if len(args) > 1 else {}} + if origin is Literal: + return {'enum': list(args)} + if isinstance(annotation, type) and issubclass(annotation, Enum): + return {'enum': [item.value for item in annotation]} + if isinstance(annotation, type) and issubclass(annotation, BaseModel): + return annotation.model_json_schema() + if isinstance(annotation, type) and dataclasses.is_dataclass(annotation): + name = f'{annotation.__module__}.{annotation.__qualname__}' + if name in seen: + return {'$ref': name} + module = sys.modules.get(annotation.__module__) + try: + hints = get_type_hints(annotation, globalns=vars(module) if module else None) + except (NameError, TypeError): + hints = annotation.__annotations__ + properties = {} + required = [] + for field in dataclasses.fields(annotation): + if not field.init or field.name.startswith('_'): + continue + properties[field.name] = _type_contract(hints.get(field.name, Any), seen | {name}) + if field.default is dataclasses.MISSING and field.default_factory is dataclasses.MISSING: + required.append(field.name) + result = {'type': 'object', 'properties': properties} + if required: + result['required'] = required + return result + primitive = {str: 'string', int: 'integer', float: 'number', bool: 'boolean'} + if annotation in primitive: + return {'type': primitive[annotation]} + return {'pythonType': getattr(annotation, '__qualname__', repr(annotation))} + + +def _parameter_contract(field: Any) -> dict[str, Any]: + annotation = getattr(field.field_info, 'annotation', field.type_) + return { + 'name': field.alias, + 'required': bool(field.required), + 'schema': _type_contract(annotation), + } + +def _extract_app_surface(app: FastAPI) -> dict[str, Any]: + """Return every route's complete request and response type shape.""" paths: dict[str, dict[str, Any]] = {} - for path, ops in (spec.get('paths') or {}).items(): - clean_ops: dict[str, Any] = {} - for method, op in ops.items(): - if method.upper() not in _HTTP_METHODS: - continue - clean_ops[method.upper()] = { - 'operationId': op.get('operationId'), - 'parameters': op.get('parameters', []), - 'responses': sorted((op.get('responses') or {}).keys()), + for route in app.routes: + if not isinstance(route, APIRoute): + continue + extra_responses = {} + for status, response in route.responses.items(): + extra_responses[str(status)] = { + 'description': response.get('description'), + 'content': response.get('content'), + 'model': _type_contract(response.get('model')) if response.get('model') else None, } - if clean_ops: - paths[path] = clean_ops - + operation = { + 'operationId': route.operation_id or route.name, + 'body': [_parameter_contract(field) for field in route.dependant.body_params], + 'path': [_parameter_contract(field) for field in route.dependant.path_params], + 'query': [_parameter_contract(field) for field in route.dependant.query_params], + 'headers': [_parameter_contract(field) for field in route.dependant.header_params], + 'cookies': [_parameter_contract(field) for field in route.dependant.cookie_params], + 'response': _type_contract(route.response_model), + 'responses': extra_responses, + 'statusCode': route.status_code or 200, + } + for method in sorted(route.methods & _HTTP_METHODS): + paths.setdefault(route.path, {})[method] = operation return {'paths': paths} def extract_full_surface() -> dict[str, Any]: - """Build all four apps and return a per-app contract surface dict.""" + """Build all five apps and return a per-app contract surface dict.""" surface: dict[str, Any] = {} for name, builder in APP_BUILDERS.items(): app = builder() diff --git a/tests/server/contract/test_client_api_contract.py b/tests/server/contract/test_client_api_contract.py index afed5e2c..ea0f743b 100644 --- a/tests/server/contract/test_client_api_contract.py +++ b/tests/server/contract/test_client_api_contract.py @@ -1,9 +1,9 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Zero-wire-change contract guard (T8.1 / R8 / Property 10). -Re-exports the OpenAPI surface of all four apps and compares it field-by-field with -the baseline captured before this spec's implementation (T0.1). The diff must be -empty. Also asserts the load-bearing invariants: ``schedule_task_and_wait`` still +Exports the request/response surface of all five apps and compares it field-by-field with +the canonical baseline. The diff must be empty. Also asserts the load-bearing +invariants: ``schedule_task_and_wait`` still exists and the only client-side additions are ``types/base.py`` and ``types/errors.py``. """ from __future__ import annotations @@ -13,11 +13,12 @@ from tests.server.contract.client_api_harness import extract_full_surface, load_baseline -def test_openapi_surface_matches_baseline(): +def test_wire_surface_matches_baseline(): current = extract_full_surface() baseline = load_baseline() + assert set(current) == {'data_plane', 'gateway', 'model', 'processor', 'sampler'} assert current == baseline, ( - 'Client-facing OpenAPI surface changed vs the pre-implementation baseline; ' + 'Client-facing wire surface changed vs the canonical baseline; ' 'this spec must be zero-wire-change. Diffing apps: ' f'{[a for a in set(current) | set(baseline) if current.get(a) != baseline.get(a)]}') diff --git a/tests/server/contract/test_error_wire.py b/tests/server/contract/test_error_wire.py new file mode 100644 index 00000000..df6be289 --- /dev/null +++ b/tests/server/contract/test_error_wire.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from fastapi import FastAPI +from fastapi.testclient import TestClient +from tinker.types import RequestFailedResponse + +from twinkle.server.gateway.tinker_handlers import _register_tinker_routes + + +class _State: + + async def get_future(self, request_id: str): + return { + 'status': 'failed', + 'result': { + 'error': 'backend timed out', + 'category': 'server', + 'error_code': 504, + 'request_id': request_id, + }, + } + + +class _Gateway: + state = _State() + + +def test_retrieve_future_returns_parseable_error_payload(): + app = FastAPI() + _register_tinker_routes(app, lambda: _Gateway()) + + response = TestClient(app).post('/retrieve_future', json={'request_id': 'req-1'}) + + assert response.status_code == 200 + body = response.json() + assert body['error_code'] == 504 + assert body['request_id'] == 'req-1' + parsed = RequestFailedResponse.model_validate(body) + assert parsed.category.value == 'server' diff --git a/tests/server/integration/test_actor_recovery.py b/tests/server/integration/test_actor_recovery.py index 2ec4f807..64696637 100644 --- a/tests/server/integration/test_actor_recovery.py +++ b/tests/server/integration/test_actor_recovery.py @@ -7,8 +7,10 @@ from __future__ import annotations import pytest +from fastapi import FastAPI from twinkle.server.model.app import ModelManagement +from twinkle.server.model.twinkle_handlers import _register_twinkle_routes class _MockModel: @@ -24,6 +26,7 @@ def ping(self) -> bool: class _HealthHarness: # Reuse the real implementations under test. + _run_model_health_probe = ModelManagement._run_model_health_probe check_model_health = ModelManagement.check_model_health mark_unhealthy = ModelManagement.mark_unhealthy _probe_after_timeout = ModelManagement._probe_after_timeout @@ -58,6 +61,20 @@ async def test_timeout_probe_marks_unhealthy_then_recovers(): assert h._model_unhealthy is False +@pytest.mark.asyncio +async def test_health_route_returns_503_when_probe_fails(): + model = _MockModel() + model.alive = False + harness = _HealthHarness(model) + app = FastAPI() + _register_twinkle_routes(app, lambda: harness) + route = next(route for route in app.routes if getattr(route, 'path', None) == '/healthz') + + response = await route.endpoint(object(), harness) + + assert response.status_code == 503 + + @pytest.mark.asyncio async def test_mark_unhealthy_is_cleared_by_successful_probe(): h = _HealthHarness(_MockModel()) diff --git a/tests/server/integration/test_blocking_boundary.py b/tests/server/integration/test_blocking_boundary.py index ddf0e5e8..6432ad79 100644 --- a/tests/server/integration/test_blocking_boundary.py +++ b/tests/server/integration/test_blocking_boundary.py @@ -9,24 +9,34 @@ from __future__ import annotations import asyncio +import threading import time from concurrent.futures import ThreadPoolExecutor +import httpx import pytest +from fastapi import FastAPI +from fastapi.responses import JSONResponse -from twinkle.server.utils.task_queue.mixin import TaskQueueMixin -from twinkle.server.utils.task_queue.types import BackendBusyError +ray = pytest.importorskip('ray') + +from twinkle.server.utils.task_queue.mixin import TaskQueueMixin # noqa: E402 +from twinkle.server.utils.task_queue.types import BackendBusyError # noqa: E402 class _Harness(TaskQueueMixin): """Minimal holder exposing the real call_backend with a chosen gate setting.""" - def __init__(self, gate_enabled: bool) -> None: - self._backend_executor = ThreadPoolExecutor(thread_name_prefix='twinkle-backend') - self._backend_admission = asyncio.Semaphore(1) if gate_enabled else None + def __init__(self, gate_enabled: bool, *, max_workers: int | None = None) -> None: + self._backend_executor = ThreadPoolExecutor( + max_workers=max_workers, thread_name_prefix='twinkle-backend') + self._backend_probe_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='twinkle-backend-probe') + self._backend_admission = asyncio.Lock() if gate_enabled else None + self._backend_poisoned = asyncio.Event() def close(self) -> None: - self._backend_executor.shutdown(wait=False) + self._backend_executor.shutdown(wait=False, cancel_futures=True) + self._backend_probe_executor.shutdown(wait=False, cancel_futures=True) @pytest.mark.asyncio @@ -50,6 +60,62 @@ async def test_healthz_style_probe_responsive_during_slow_backend(): h.close() +@pytest.mark.asyncio +async def test_normal_gate_contention_waits_instead_of_failing(): + h = _Harness(gate_enabled=True) + try: + first = asyncio.create_task(h.call_backend(lambda: (time.sleep(0.2), 'first')[1])) + await asyncio.sleep(0.05) + second = asyncio.create_task(h.call_backend(lambda: 'second')) + assert await first == 'first' + assert await second == 'second' + finally: + h.close() + + +@pytest.mark.asyncio +async def test_cancelled_gate_waiter_does_not_steal_lock(): + h = _Harness(gate_enabled=True) + release = threading.Event() + + def wait_for_release(): + while not release.is_set(): + time.sleep(0.01) + + try: + first = asyncio.create_task(h.call_backend(wait_for_release)) + await asyncio.sleep(0.05) + waiter = asyncio.create_task(h.call_backend(lambda: 'cancelled')) + await asyncio.sleep(0.05) + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + release.set() + await first + assert await h.call_backend(lambda: 'next') == 'next' + finally: + release.set() + h.close() + + +@pytest.mark.asyncio +async def test_cancelled_queued_backend_call_releases_gate(): + h = _Harness(gate_enabled=True, max_workers=1) + release_worker = threading.Event() + occupied = h._backend_executor.submit(release_worker.wait) + try: + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(h.call_backend(lambda: 'never-started'), timeout=0.05) + await asyncio.sleep(0) + assert not h._backend_admission.locked() + release_worker.set() + occupied.result(timeout=5) + assert await h.call_backend(lambda: 'next') == 'next' + finally: + release_worker.set() + h.close() + + @pytest.mark.asyncio async def test_gate_held_by_leaked_call_fast_fails_next_task(): """Property 4 / R2#4: a call that outlives its wait_for keeps the gate; the next @@ -81,6 +147,49 @@ def would_enter_backend(): h.close() +@pytest.mark.asyncio +async def test_probe_times_out_while_same_serial_actor_is_busy(): + + @ray.remote + class SerialActor: + + def slow(self): + time.sleep(1.0) + + def ping(self): + return True + + started_ray = not ray.is_initialized() + if started_ray: + ray.init(num_cpus=1, logging_level='ERROR') + actor = SerialActor.remote() + h = _Harness(gate_enabled=True) + app = FastAPI() + + @app.get('/healthz') + async def healthz(): + try: + await h.call_backend(lambda: ray.get(actor.ping.remote(), timeout=0.2), admit=False) + return {'healthy': True} + except ray.exceptions.GetTimeoutError: + return JSONResponse(status_code=503, content={'healthy': False}) + + try: + slow = asyncio.create_task(h.call_backend(lambda: ray.get(actor.slow.remote(), timeout=2))) + await asyncio.sleep(0.1) + start = time.monotonic() + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url='http://test') as client: + response = await client.get('/healthz') + assert response.status_code == 503 + assert time.monotonic() - start < 5 + await slow + finally: + h.close() + ray.kill(actor) + if started_ray: + ray.shutdown() + + @pytest.mark.asyncio async def test_sampler_without_gate_runs_two_calls_concurrently(): """R9#2 case 3 / opt-in: with the gate disabled (SamplerManagement), two backend diff --git a/tests/server/integration/test_nccl_safe_tinker_e2e.py b/tests/server/integration/test_nccl_safe_tinker_e2e.py index b47666b9..4846a2bc 100644 --- a/tests/server/integration/test_nccl_safe_tinker_e2e.py +++ b/tests/server/integration/test_nccl_safe_tinker_e2e.py @@ -9,7 +9,7 @@ Prerequisites: 1. Ray cluster running with GPUs (2 for model DP/TP) - 2. Twinkle server started (no fault-tolerance env switch exists any more) + 2. Twinkle server started with queue_config.execution_timeout=30 Usage (pytest, requires TWINKLE_TEST_GPU_E2E=1): TWINKLE_TEST_GPU_E2E=1 pytest tests/server/integration/test_nccl_safe_tinker_e2e.py -v @@ -29,7 +29,8 @@ BASE_MODEL = 'Qwen/Qwen3.5-4B' SERVER_URL = os.environ.get('TWINKLE_SERVER_URL', 'http://localhost:9000') -TIMEOUT = 120 +EXECUTION_TIMEOUT = float(os.environ.get('TWINKLE_TEST_EXECUTION_TIMEOUT', '30')) +TIMEOUT = EXECUTION_TIMEOUT + 15 def _init_client(): @@ -71,13 +72,15 @@ def test_failure_is_terminal_then_valid_request_succeeds(): necessary, not optional. """ from tinker import types + from tinker._exceptions import RequestFailedError tc = _init_client() # Deliberately malformed: logprobs length inconsistent with the completion. bad = [_make_datum(bad_logprobs_len=5) for _ in range(4)] start = time.time() - with pytest.raises(Exception): # RequestFailedError or a raised failed terminal - tc.forward_backward(bad, 'importance_sampling').result() + with pytest.raises(RequestFailedError) as caught: + tc.forward_backward(bad, 'importance_sampling').result(timeout=TIMEOUT) + assert caught.value.category is types.RequestErrorCategory.Server assert time.time() - start < TIMEOUT, 'malformed request must fail fast, not hang (NCCL)' # Recovery: a subsequent valid request on the same deployment must succeed. @@ -85,3 +88,22 @@ def test_failure_is_terminal_then_valid_request_succeeds(): result = tc.forward_backward(good, 'importance_sampling').result() assert result is not None tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() + + +def test_partial_rank_failure_is_terminal_then_recovers(): + from tinker import types + from tinker._exceptions import RequestFailedError + tc = _init_client() + + batch = [_make_datum() for _ in range(4)] + batch[0] = _make_datum(bad_logprobs_len=5) + start = time.time() + with pytest.raises(RequestFailedError) as caught: + tc.forward_backward(batch, 'importance_sampling').result(timeout=TIMEOUT) + assert caught.value.category is types.RequestErrorCategory.Server + assert 'global_rank=' in str(caught.value) + assert time.time() - start < TIMEOUT + + result = tc.forward_backward([_make_datum() for _ in range(4)], 'importance_sampling').result() + assert result is not None + tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() diff --git a/tests/server/integration/test_nccl_safe_twinkle_e2e.py b/tests/server/integration/test_nccl_safe_twinkle_e2e.py index 90a33cf3..5eed7141 100644 --- a/tests/server/integration/test_nccl_safe_twinkle_e2e.py +++ b/tests/server/integration/test_nccl_safe_twinkle_e2e.py @@ -8,7 +8,7 @@ Prerequisites: 1. Ray cluster running with GPUs (2 for model DP/TP) - 2. Twinkle server started (no fault-tolerance env switch exists any more) + 2. Twinkle server started with queue_config.execution_timeout=30 Usage (pytest, requires TWINKLE_TEST_GPU_E2E=1): TWINKLE_TEST_GPU_E2E=1 pytest tests/server/integration/test_nccl_safe_twinkle_e2e.py -v @@ -28,7 +28,8 @@ BASE_MODEL = 'Qwen/Qwen3.5-4B' SERVER_URL = os.environ.get('TWINKLE_SERVER_URL', 'http://localhost:9000') -TIMEOUT = 120 +EXECUTION_TIMEOUT = float(os.environ.get('TWINKLE_TEST_EXECUTION_TIMEOUT', '30')) +TIMEOUT = EXECUTION_TIMEOUT + 15 ADAPTER_NAME = 'loud-failure-test' @@ -79,9 +80,10 @@ def test_failure_is_terminal_then_valid_request_succeeds(): bad_features, bad_old_logps, bad_adv = _make_inputs(bad_old_logps_len=5) start = time.time() - with pytest.raises(Exception): + with pytest.raises(Exception) as caught: model.forward_backward( inputs=bad_features, adapter_name=ADAPTER_NAME, old_logps=bad_old_logps, advantages=bad_adv) + assert 'global_rank=' in str(caught.value) assert time.time() - start < TIMEOUT, 'malformed request must fail fast, not hang (NCCL)' good_features, good_old_logps, good_adv = _make_inputs() diff --git a/tests/server/model/test_replica_lifecycle.py b/tests/server/model/test_replica_lifecycle.py index e42193c3..84b96139 100644 --- a/tests/server/model/test_replica_lifecycle.py +++ b/tests/server/model/test_replica_lifecycle.py @@ -12,12 +12,17 @@ class _CapacityState: def __init__(self) -> None: self.capacities: dict[str, int] = {} + self.last_seen: set[str] = set() async def register_replica(self, replica_id: str, max_loras: int) -> None: self.capacities[replica_id] = max_loras async def unregister_replica(self, replica_id: str) -> None: self.capacities.pop(replica_id, None) + self.last_seen.discard(replica_id) + + async def touch_replica_last_seen(self, replica_id: str) -> None: + self.last_seen.add(replica_id) async def get_capacity_info(self) -> dict[str, int]: max_loras = sum(self.capacities.values()) @@ -31,6 +36,7 @@ def _make_lifecycle_manager(state: _CapacityState, replica_id: str, max_loras: i manager.max_loras = max_loras manager._replica_registered = False manager.data_plane = SimpleNamespace(close=AsyncMock()) + manager.shutdown_task_queue = AsyncMock() return manager @@ -42,6 +48,7 @@ async def test_replica_lifecycle_updates_shared_capacity() -> None: await first._register_replica_on_startup() assert await state.get_capacity_info() == {'max_loras': 3, 'used_loras': 0, 'free_loras': 3} + assert state.last_seen == {'replica-1'} await second._register_replica_on_startup() assert await state.get_capacity_info() == {'max_loras': 6, 'used_loras': 0, 'free_loras': 6} @@ -52,9 +59,10 @@ async def test_replica_lifecycle_updates_shared_capacity() -> None: @pytest.mark.asyncio async def test_async_constructor_registers_replica_before_ready() -> None: - state = SimpleNamespace(register_replica=AsyncMock()) + state = SimpleNamespace(register_replica=AsyncMock(), touch_replica_last_seen=AsyncMock()) replica_context = SimpleNamespace(replica_id=SimpleNamespace(unique_id='replica-1')) manager = ModelManagement.__new__(ModelManagement) + manager._task_queue_config = SimpleNamespace(effective_execution_timeout=1800.0) with patch('twinkle.server.model.app.DeviceGroup', return_value=SimpleNamespace(name='group')), \ patch('twinkle.server.model.app.init_twinkle_runtime', return_value=None), \ @@ -75,4 +83,5 @@ async def test_async_constructor_registers_replica_before_ready() -> None: ) state.register_replica.assert_awaited_once_with('replica-1', 3) + state.touch_replica_last_seen.assert_awaited_once_with('replica-1') assert manager._replica_registered is True diff --git a/tests/server/model/test_tinker_handlers.py b/tests/server/model/test_tinker_handlers.py index 474ce700..66a9ff98 100644 --- a/tests/server/model/test_tinker_handlers.py +++ b/tests/server/model/test_tinker_handlers.py @@ -73,9 +73,11 @@ def assert_resource_exists(self, adapter_name): pass async def schedule_task(self, task, **kwargs): - # Actually execute the task to test response logic return await task() + async def call_backend(self, fn, /, *args, **kwargs): + return fn(*args, **kwargs) + @pytest.mark.asyncio @patch('twinkle.server.model.tinker_handlers.create_checkpoint_manager') diff --git a/tests/server/sampler/test_stream_guarantees.py b/tests/server/sampler/test_stream_guarantees.py new file mode 100644 index 00000000..d1489575 --- /dev/null +++ b/tests/server/sampler/test_stream_guarantees.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import asyncio +import json +import threading +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from fastapi import FastAPI + +from twinkle.server.sampler.app import SamplerManagement +from twinkle.server.sampler.twinkle_handlers import _await_generation, _register_twinkle_sampler_routes, _stream_queue + + +class _BlockingQueue: + + def __init__(self) -> None: + self.released = threading.Event() + self.get_exited = threading.Event() + self.closed = False + + def get(self): + self.released.wait() + self.get_exited.set() + return 'sentinel' + + def shutdown(self, *, force: bool) -> None: + assert force is True + self.closed = True + self.released.set() + + +@pytest.mark.asyncio +async def test_sampler_request_refreshes_replica_liveness(): + service = SamplerManagement.__new__(SamplerManagement) + service.replica_id = 'sampler-replica' + service.state = SimpleNamespace(touch_replica_last_seen=AsyncMock()) + service._ensure_sticky = AsyncMock() + service._ensure_state_cleanup_started = AsyncMock() + request = SimpleNamespace( + headers={'Authorization': 'Bearer token'}, state=SimpleNamespace(token='token')) + + assert await service._on_request_start(request) == 'token' + service.state.touch_replica_last_seen.assert_awaited_once_with('sampler-replica') + + +@pytest.mark.asyncio +async def test_stream_without_actor_returns_structured_error(): + service = SimpleNamespace( + sampler=SimpleNamespace(_actors=[]), + _on_request_start=AsyncMock(return_value='token'), + ) + app = FastAPI() + _register_twinkle_sampler_routes(app, lambda: service) + route = next(route for route in app.routes if getattr(route, 'path', None) == '/twinkle/sample_stream') + request = SimpleNamespace(state=SimpleNamespace(request_id='request')) + body = SimpleNamespace(adapter_name='', adapter_uri=None, inputs={'input_ids': [1]}, sampling_params=None) + + response = await route.endpoint(request, body, service) + chunks = [chunk async for chunk in response.body_iterator] + payload = json.loads(chunks[0]) + assert payload['category'] == 'server' + assert payload['error_code'] == 503 + assert payload['request_id'].startswith('req_') + + +@pytest.mark.asyncio +async def test_stream_timeout_returns_error_payload_and_closes_queue(): + queue = _BlockingQueue() + chunks = [ + chunk async for chunk in _stream_queue( + queue, + sentinel='sentinel', + request_id='req-stream', + total_timeout=0.05, + single_get_timeout=0.05, + ) + ] + + payload = json.loads(chunks[0]) + assert payload['category'] == 'server' + assert payload['error_code'] == 504 + assert payload['request_id'] == 'req-stream' + assert queue.closed is True + assert queue.get_exited.wait(timeout=5) + + +class _GenerationService: + + def __init__(self) -> None: + self.cancelled = False + self.sampler = SimpleNamespace( + get_generation_status=lambda _submission_id: {'status': 'running'}, + collect_generation=lambda _submission_id: [], + cancel_generation=self._cancel, + ) + + async def call_backend(self, fn, /, *args, **kwargs): + return fn(*args, **kwargs) + + def _cancel(self, _submission_id: str) -> None: + self.cancelled = True + + +@pytest.mark.asyncio +async def test_generation_poll_has_total_timeout_and_cancels(): + service = _GenerationService() + + with pytest.raises(asyncio.TimeoutError): + await _await_generation(service, 'submission', timeout=0.05) + + assert service.cancelled is True diff --git a/tests/server/sampler/test_tinker_handlers.py b/tests/server/sampler/test_tinker_handlers.py index 1e338594..77a15ce6 100644 --- a/tests/server/sampler/test_tinker_handlers.py +++ b/tests/server/sampler/test_tinker_handlers.py @@ -52,6 +52,9 @@ async def _on_request_start(self, request): async def schedule_task(self, task, **kwargs): return await task() + async def call_backend(self, fn, /, *args, **kwargs): + return fn(*args, **kwargs) + @pytest.mark.asyncio async def test_tinker_asample_allows_base_model_session_without_model_path(): @@ -71,4 +74,6 @@ async def test_tinker_asample_allows_base_model_session_without_model_path(): response = await route.endpoint(request, body, management) assert isinstance(response, types.SampleResponse) + assert response.sequences[0].tokens == [1, 2] + assert response.sequences[0].sequence_id assert management.sampler.adapter_paths == [None] diff --git a/tests/server/state/test_error_payload.py b/tests/server/state/test_error_payload.py index b5483445..eb224f73 100644 --- a/tests/server/state/test_error_payload.py +++ b/tests/server/state/test_error_payload.py @@ -5,6 +5,9 @@ """ from __future__ import annotations +import pytest +from pydantic import ValidationError + from twinkle.server.utils.task_errors import error_payload_from_stored, task_error_payload from twinkle_client.types.errors import ErrorCategory, ErrorPayload @@ -41,7 +44,13 @@ def test_user_category_carries_no_traceback(): category=ErrorCategory.User, traceback_text='Traceback (most recent call last): ...') assert payload['category'] == ErrorCategory.User.value - assert payload['traceback'] is None + assert 'traceback' not in payload + + +def test_error_category_matches_tinker_wire_values(): + from tinker.types import RequestErrorCategory + + assert {item.value for item in RequestErrorCategory} == {item.value for item in ErrorCategory} def test_tinker_sdk_parses_six_field_like_two_field(): @@ -54,10 +63,35 @@ def test_tinker_sdk_parses_six_field_like_two_field(): from tinker.types import RequestFailedResponse two = {'error': 'boom', 'category': 'server'} - six = {**two, 'error_code': 504, 'request_id': 'req_9', 'traceback': None, 'details': None} + six = task_error_payload('boom', request_id='req_9', error_code=504) parsed_six = RequestFailedResponse.model_validate(six) parsed_two = RequestFailedResponse.model_validate(two) assert parsed_six.error == parsed_two.error assert parsed_six.category == parsed_two.category + + +def test_legacy_title_case_category_is_normalized(): + payload = error_payload_from_stored({'error': 'boom', 'category': 'Server'}, request_id='req_10') + assert payload.category is ErrorCategory.Server + assert payload.category.value == 'server' + + +@pytest.mark.parametrize('category', [ErrorCategory.User, ErrorCategory.Unknown]) +def test_non_server_traceback_is_rejected(category): + with pytest.raises(ValidationError): + ErrorPayload( + error='bad input', + category=category, + error_code=400, + request_id='req_11', + traceback='server stack', + ) + + +def test_legacy_unknown_traceback_is_removed(): + payload = error_payload_from_stored( + {'error': 'legacy', 'category': 'Unknown', 'traceback': 'old stack'}, request_id='req_12') + assert payload.category is ErrorCategory.Unknown + assert payload.traceback is None diff --git a/tests/server/state/test_future_lifecycle.py b/tests/server/state/test_future_lifecycle.py index c32a6172..55905afa 100644 --- a/tests/server/state/test_future_lifecycle.py +++ b/tests/server/state/test_future_lifecycle.py @@ -20,15 +20,21 @@ def manager(tmp_path): return FutureManager(backend, expiration_timeout=300.0) -async def _store(manager, request_id, status, *, replica_id=None): - await manager.store_status(request_id, status, model_id='m1', replica_id=replica_id) +async def _store(manager, request_id, status, *, replica_id=None, absolute_deadline=None): + await manager.store_status( + request_id, + status, + model_id='m1', + replica_id=replica_id, + absolute_deadline=absolute_deadline, + ) @pytest.mark.asyncio async def test_non_terminal_with_live_replica_is_kept(manager): await _store(manager, 'r1', 'running', replica_id='replica-A') removed = await manager.cleanup_expired( - cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) + cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}) assert removed == 0 rec = await manager.get('r1') assert rec is not None and rec.status == 'running' @@ -37,27 +43,35 @@ async def test_non_terminal_with_live_replica_is_kept(manager): @pytest.mark.asyncio async def test_non_terminal_orphan_is_failed_not_deleted(manager): await _store(manager, 'r2', 'running', replica_id='dead-replica') - await manager.cleanup_expired(cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) + await manager.cleanup_expired(cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}) rec = await manager.get('r2') assert rec is not None # NOT deleted (Property 6) assert rec.status == 'failed' - assert rec.result['category'] == 'Server' + assert rec.result['category'] == 'server' @pytest.mark.asyncio -async def test_non_terminal_over_absolute_ttl_is_failed(manager): - await _store(manager, 'r3', 'running', replica_id='replica-A') - # absolute_ttl=0 makes any positive age exceed the bound. - await manager.cleanup_expired(cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}, absolute_ttl=0.0) +async def test_non_terminal_past_absolute_deadline_is_failed(manager): + await _store(manager, 'r3', 'running', replica_id='replica-A', absolute_deadline=time.time() - 1) + await manager.cleanup_expired(cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}) rec = await manager.get('r3') assert rec is not None and rec.status == 'failed' +@pytest.mark.asyncio +async def test_legacy_record_without_deadline_uses_expiration_timeout(manager): + await _store(manager, 'legacy', 'running', replica_id=None) + with mock.patch('twinkle.server.state.future_manager.time.time', return_value=time.time() + 301): + await manager.cleanup_expired(cutoff_time=time.time() + 10, alive_replica_ids=set()) + rec = await manager.get('legacy') + assert rec is not None and rec.status == 'failed' + + @pytest.mark.asyncio async def test_terminal_expired_is_deleted(manager): await _store(manager, 'r4', 'completed', replica_id='replica-A') removed = await manager.cleanup_expired( - cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}, absolute_ttl=None) + cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}) assert removed == 1 assert await manager.get('r4') is None @@ -83,11 +97,14 @@ async def test_terminal_to_terminal_same_is_dropped_without_warning(manager): @pytest.mark.asyncio -async def test_replica_id_set_at_creation_not_overwritten(manager): - await _store(manager, 'r7', 'pending', replica_id='replica-A') - await manager.store_status('r7', 'running', model_id='m1', replica_id='replica-B') +async def test_replica_id_and_deadline_set_at_creation_not_overwritten(manager): + deadline = time.time() + 100 + await _store(manager, 'r7', 'pending', replica_id='replica-A', absolute_deadline=deadline) + await manager.store_status( + 'r7', 'running', model_id='m1', replica_id='replica-B', absolute_deadline=time.time() + 999) rec = await manager.get('r7') - assert rec.replica_id == 'replica-A' # creation value preserved + assert rec.replica_id == 'replica-A' + assert rec.absolute_deadline == deadline @pytest.mark.asyncio diff --git a/tests/server/state/test_managers.py b/tests/server/state/test_managers.py index 1b9016be..63fc24be 100644 --- a/tests/server/state/test_managers.py +++ b/tests/server/state/test_managers.py @@ -172,6 +172,12 @@ async def test_replica_registration(self, manager): assert info['used_loras'] == 0 assert info['free_loras'] == 5 + @pytest.mark.asyncio + async def test_liveness_only_replica_is_alive(self, manager): + await manager.touch_replica_last_seen('sampler-replica') + alive = await manager.get_alive_replica_ids(liveness_threshold=60) + assert 'sampler-replica' in alive + @pytest.mark.asyncio async def test_capacity_info_after_add(self, manager): await manager.register_replica('r1', max_loras=3) diff --git a/tests/server/static/backend_call_exemptions.py b/tests/server/static/backend_call_exemptions.py index 1ba0674e..4dcd3bf0 100644 --- a/tests/server/static/backend_call_exemptions.py +++ b/tests/server/static/backend_call_exemptions.py @@ -19,5 +19,5 @@ # (module_relpath under src/twinkle/server, innermost enclosing function name) BACKEND_CALL_EXEMPTIONS: frozenset[tuple[str, str]] = frozenset({ - ('sampler/twinkle_handlers.py', '_stream_generator'), + ('sampler/twinkle_handlers.py', '_stream_queue'), }) diff --git a/tests/server/static/test_no_direct_backend_call.py b/tests/server/static/test_no_direct_backend_call.py index 4e2750ee..5b218e91 100644 --- a/tests/server/static/test_no_direct_backend_call.py +++ b/tests/server/static/test_no_direct_backend_call.py @@ -3,10 +3,9 @@ Spec: T3.7 / R2#7 / R2#8 / Property 2. -Asserts that no module under ``src/twinkle/server`` invokes ``self.model.(...)`` -or ``self.sampler.(...)`` directly -- every such call must go through -``call_backend`` (the Blocking_Call_Boundary). The scan range is the directory -(not a file list), so a newly added handler file cannot silently escape it. +Asserts that backend methods are invoked only through ``call_backend``. The check +covers direct calls, aliases created with ``getattr``, and methods passed to generic +thread executors. The scan range is the server directory rather than a file list. Allowed bypasses are read from the shared ``backend_call_exemptions`` module, which this spec and ``server-request-lifecycle`` are meant to consume unchanged. This spec @@ -24,14 +23,22 @@ _SERVER_ROOT = pathlib.Path(twinkle.__file__).resolve().parent / 'server' -def _is_self_backend_call(node: ast.Call) -> str | None: - """Return 'model'/'sampler' if node is a direct self.model/self.sampler.() call.""" - func = node.func - if not isinstance(func, ast.Attribute): +def _backend_method(node: ast.AST) -> str | None: + if not isinstance(node, ast.Attribute): return None - owner = func.value # the object the method is called on - if (isinstance(owner, ast.Attribute) and owner.attr in ('model', 'sampler') - and isinstance(owner.value, ast.Name) and owner.value.id == 'self'): + owner = node.value + if isinstance(owner, ast.Attribute) and owner.attr in ('model', 'sampler'): + return owner.attr + return None + + +def _getattr_backend_method(node: ast.AST) -> str | None: + if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Name) or node.func.id != 'getattr': + return None + if not node.args: + return None + owner = node.args[0] + if isinstance(owner, ast.Attribute) and owner.attr in ('model', 'sampler'): return owner.attr return None @@ -41,6 +48,7 @@ class _Collector(ast.NodeVisitor): def __init__(self, relpath: str) -> None: self.relpath = relpath self.func_stack: list[str] = [] + self.backend_aliases: set[str] = set() self.offenders: list[tuple[str, str, int, str]] = [] def _visit_func(self, node: ast.AST) -> None: @@ -51,12 +59,26 @@ def _visit_func(self, node: ast.AST) -> None: visit_FunctionDef = _visit_func visit_AsyncFunctionDef = _visit_func + def visit_Assign(self, node: ast.Assign) -> None: + if _getattr_backend_method(node.value) is not None: + self.backend_aliases.update(target.id for target in node.targets if isinstance(target, ast.Name)) + self.generic_visit(node) + def visit_Call(self, node: ast.Call) -> None: - owner = _is_self_backend_call(node) + owner = _backend_method(node.func) + label = ast.unparse(node.func) if owner is not None else None + if isinstance(node.func, ast.Name) and node.func.id in self.backend_aliases: + owner = 'alias' + label = node.func.id + if isinstance(node.func, ast.Attribute) and node.func.attr in ('to_thread', 'run_in_executor') and node.args: + escaped_owner = _backend_method(node.args[0]) or _getattr_backend_method(node.args[0]) + if escaped_owner is not None: + owner = escaped_owner + label = f'{ast.unparse(node.func)}({ast.unparse(node.args[0])})' if owner is not None: enclosing = self.func_stack[-1] if self.func_stack else '' if (self.relpath, enclosing) not in BACKEND_CALL_EXEMPTIONS: - self.offenders.append((self.relpath, enclosing, node.lineno, f'self.{owner}.{node.func.attr}')) + self.offenders.append((self.relpath, enclosing, node.lineno, label or owner)) self.generic_visit(node) @@ -74,5 +96,27 @@ def test_no_direct_backend_call_in_server(): def test_exemptions_are_read_from_shared_file(): - # The shared file is the single source of allowed bypasses (R2#8). - assert ('sampler/twinkle_handlers.py', '_stream_generator') in BACKEND_CALL_EXEMPTIONS + assert ('sampler/twinkle_handlers.py', '_stream_queue') in BACKEND_CALL_EXEMPTIONS + + +def test_checker_detects_indirect_backend_calls(): + source = """ +async def route(self): + await asyncio.to_thread(self.model.save) + unload = getattr(self.sampler, 'unload_adapter_paths') + unload([]) +""" + collector = _Collector('example.py') + collector.visit(ast.parse(source)) + assert len(collector.offenders) == 2 + + +def test_checker_allows_call_backend(): + source = """ +async def route(self): + unload = getattr(self.sampler, 'unload_adapter_paths') + await self.call_backend(unload, []) +""" + collector = _Collector('example.py') + collector.visit(ast.parse(source)) + assert collector.offenders == [] diff --git a/tests/server/utils/task_queue/test_config.py b/tests/server/utils/task_queue/test_config.py index 3126c507..878bfa12 100644 --- a/tests/server/utils/task_queue/test_config.py +++ b/tests/server/utils/task_queue/test_config.py @@ -22,6 +22,7 @@ 'tps_limit': 16000.0, 'window_seconds': 1.0, 'queue_timeout': 300.0, + 'execution_timeout': 1800.0, 'token_cleanup_interval': 60.0, 'max_input_tokens': 16000, } @@ -113,3 +114,15 @@ def test_extra_field_rejected() -> None: """``extra='forbid'`` rejects unknown keys.""" with pytest.raises(ValidationError): TaskQueueConfig(unknown_field=1) + + +def test_zero_execution_timeout_uses_finite_fallback() -> None: + assert TaskQueueConfig(execution_timeout=0).effective_execution_timeout == 3600 + + +def test_absolute_future_ttl_uses_conservative_backend_bound() -> None: + config = TaskQueueConfig(queue_timeout=10, execution_timeout=20) + assert config.absolute_future_ttl(collect_width=2) == 2 * (10 + 2 * 3600) + + config = TaskQueueConfig(queue_timeout=10, execution_timeout=5000) + assert config.absolute_future_ttl(collect_width=2) == 2 * (10 + 2 * 5000) diff --git a/tests/server/utils/test_task_errors.py b/tests/server/utils/test_task_errors.py index 698ad5b7..832ca109 100644 --- a/tests/server/utils/test_task_errors.py +++ b/tests/server/utils/test_task_errors.py @@ -11,7 +11,7 @@ def test_task_error_payload_builds_error_payload_dict(): assert payload['category'] == ErrorCategory.Server.value assert payload['error_code'] == 500 assert payload['request_id'] == 'req_1' - assert payload['traceback'] is None + assert 'traceback' not in payload def test_task_error_payload_user_category_drops_traceback(): @@ -19,7 +19,13 @@ def test_task_error_payload_user_category_drops_traceback(): 'bad input', request_id='req_2', error_code=400, category=ErrorCategory.User, traceback_text='Traceback...') assert payload['category'] == ErrorCategory.User.value - assert payload['traceback'] is None + assert 'traceback' not in payload + + +def test_error_summary_is_single_line(): + payload = task_error_payload( + 'RuntimeError: boom\n File "/server/path.py", line 1', request_id='req-lines') + assert payload['error'] == 'RuntimeError: boom' def test_error_payload_from_stored_backfills_two_field_legacy(): diff --git a/tests/server/utils/test_task_queue_mixin.py b/tests/server/utils/test_task_queue_mixin.py index f0bdbf96..04200285 100644 --- a/tests/server/utils/test_task_queue_mixin.py +++ b/tests/server/utils/test_task_queue_mixin.py @@ -4,6 +4,7 @@ from twinkle.server.utils.task_queue.config import TaskQueueConfig from twinkle.server.utils.task_queue.mixin import TaskQueueMixin +from twinkle.server.utils.task_queue.types import UserTaskError from twinkle.server.utils.task_queue.worker import ComputeWorker @@ -57,7 +58,7 @@ async def test_preflight_rejects_batch_without_per_dp_multiple(): assert result == {'request_id': 'req1', 'model_id': 'model1'} _, kwargs = queue.state.records[-1] - assert kwargs['result']['category'] == 'User' + assert kwargs['result']['category'] == 'user' assert 'Batch size 2 must be divisible by 4' in kwargs['result']['error'] @@ -93,6 +94,7 @@ async def work(): await asyncio.sleep(0) assert [args[1] for args, _ in queue.state.records] == ['running', 'completed'] + assert queue.state.records[0][1]['absolute_deadline'] > 0 assert queue.state.records[-1][1]['result'] == {'ok': True} @@ -122,6 +124,7 @@ async def work(): @pytest.mark.asyncio async def test_polling_schedule_task_still_persists_its_result(): queue = _DummyQueue() + queue.replica_id = 'replica-1' queue.enable_compute_worker() result = {'value': 42} @@ -142,6 +145,9 @@ async def work(): finally: await queue._compute_worker.stop() + pending = next(kwargs for args, kwargs in queue.state.records if args[1] == 'pending') + assert pending['replica_id'] == 'replica-1' + assert pending['absolute_deadline'] > 0 assert completed[-1]['result'] is result @@ -167,6 +173,28 @@ async def work(): assert queue.state.records == [] +@pytest.mark.asyncio +async def test_user_task_error_is_stored_as_user_failure(): + queue = _DummyQueue() + queue.enable_compute_worker() + + async def work(): + raise UserTaskError('invalid request') + + try: + await queue.schedule_task(work, model_id='model1', token='token1') + for _ in range(100): + failed = [kwargs for args, kwargs in queue.state.records if args[1] == 'failed'] + if failed: + break + await asyncio.sleep(0) + finally: + await queue._compute_worker.stop() + + assert failed[-1]['result']['category'] == 'user' + assert 'traceback' not in failed[-1]['result'] + + @pytest.mark.asyncio async def test_schedule_task_and_wait_reports_preflight_failure_without_persisting_it(): queue = _DummyQueue() diff --git a/tests/twinkle_client/test_types_contract.py b/tests/twinkle_client/test_types_contract.py index 877f2bf9..3b048c61 100644 --- a/tests/twinkle_client/test_types_contract.py +++ b/tests/twinkle_client/test_types_contract.py @@ -4,14 +4,9 @@ - T6.2 / R7#9: ``QueueStateLiteral`` value set equals the server ``QueueState`` enum. - T6.3 / R7#7: naming disambiguation guard. -Note on T6.3 scope (agreed deviation from the literal wording): ``twinkle_client`` is -a tinker-compatible client and already shares many public class names with -``tinker.types`` at HEAD (``ForwardRequest``, ``SampleRequest``, ``Checkpoint`` ...), -so a literal "the two modules' public export names are disjoint" assertion cannot -hold and twinkle's names are intentionally NOT renamed. Instead this guard enforces -the invariant the ruling actually targets: no ``src/twinkle/**`` module may bind a -tinker type and a twinkle_client type to the *same local name* (tinker must be -aliased when both coexist) -- which is what makes a review diff misread possible. +The two SDKs already share public names. The contract freezes that legacy set and +rejects new collisions while requiring explicit aliases when both SDKs are imported +in one module. """ from __future__ import annotations @@ -24,6 +19,26 @@ from twinkle_client.types.errors import QueueStateLiteral _TWINKLE_SRC = pathlib.Path(twinkle.__file__).resolve().parent +_LEGACY_PUBLIC_NAME_OVERLAP = frozenset({ + 'Checkpoint', + 'CheckpointsListResponse', + 'CreateModelRequest', + 'CreateSessionRequest', + 'CreateSessionResponse', + 'Cursor', + 'ForwardRequest', + 'GetServerCapabilitiesResponse', + 'HealthResponse', + 'LoraConfig', + 'SampleRequest', + 'SessionHeartbeatRequest', + 'SessionHeartbeatResponse', + 'SupportedModel', + 'TrainingRun', + 'TrainingRunsResponse', + 'WeightsInfoResponse', + 'checkpoint', +}) def test_queue_state_literal_matches_server_enum(): @@ -66,6 +81,18 @@ def _binding_collisions(tree: ast.AST) -> set[str]: return tinker_names & twinkle_names +def test_public_name_overlap_does_not_grow(): + import tinker.types + import twinkle_client.types + + overlap = { + name + for name in set(dir(tinker.types)) & set(dir(twinkle_client.types)) + if not name.startswith('_') + } + assert overlap == _LEGACY_PUBLIC_NAME_OVERLAP + + def test_no_tinker_twinkle_same_name_binding(): offenders: dict[str, set[str]] = {} for path in _TWINKLE_SRC.rglob('*.py'): diff --git a/tests/utils/test_nccl_safe.py b/tests/utils/test_nccl_safe.py new file mode 100644 index 00000000..62b0d8e1 --- /dev/null +++ b/tests/utils/test_nccl_safe.py @@ -0,0 +1,18 @@ +from unittest.mock import patch + +import pytest + +from twinkle.utils.nccl_safe import nccl_safe_megatron + + +def test_nccl_failure_preserves_type_and_adds_rank_context(): + + @nccl_safe_megatron + def fail(_self): + raise ValueError('bad shape') + + with patch('twinkle.utils.nccl_safe._global_rank', return_value=3): + with pytest.raises(ValueError) as caught: + fail(object()) + + assert 'global_rank=3' in ''.join(getattr(caught.value, '__notes__', caught.value.args)) From d56bcca38c484548e54ec8cc6205e583feae6288 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Wed, 16 Sep 2026 01:01:43 +0800 Subject: [PATCH 13/32] fix: revert tinker to 0.16.1 and keep request types FastAPI-compatible tinker 0.29.0 turned its tensor request types into dataclasses, which breaks FastAPI OpenAPI generation. Revert the pin and the code that had been migrated to the 0.29.0 API: - pyproject: tinker 0.29.0 -> 0.16.1; restore poetry.lock to match - sampler/tinker_handlers: use 0.16.1 SampledSequence/SampleResponse fields - contract harness: fastapi 0.136 ModelField compat; normalize :path route converters so the baseline stores client-facing paths - regenerate client_api_baseline.json against tinker 0.16.1 - test fixtures: add call_backend/_task_queue_config; drop 0.29-only assertion --- poetry.lock | 2935 ++--------------- pyproject.toml | 2 +- src/twinkle/server/sampler/tinker_handlers.py | 10 +- .../server/contract/client_api_baseline.json | 1648 +++++---- tests/server/contract/client_api_harness.py | 10 +- .../server/model/test_twinkle_async_inputs.py | 3 + tests/server/sampler/test_tinker_handlers.py | 1 - .../server/sampler/test_twinkle_async_rows.py | 6 + 8 files changed, 1284 insertions(+), 3331 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4a7b37e2..773c9b14 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,17 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. - -[[package]] -name = "absl-py" -version = "2.5.0" -description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "absl_py-2.5.0-py3-none-any.whl", hash = "sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba"}, - {file = "absl_py-2.5.0.tar.gz", hash = "sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f"}, -] +# This file is automatically @generated by Poetry 2.3.2 and should not be changed by hand. [[package]] name = "accelerate" @@ -223,7 +210,6 @@ files = [ [package.dependencies] aiohappyeyeballs = ">=2.5.0" aiosignal = ">=1.4.0" -async-timeout = {version = ">=4.0,<6.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" @@ -240,7 +226,7 @@ description = "CORS support for aiohttp" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "aiohttp_cors-0.8.1-py3-none-any.whl", hash = "sha256:3180cf304c5c712d626b9162b195b1db7ddf976a2a25172b35bb2448b890a80d"}, {file = "aiohttp_cors-0.8.1.tar.gz", hash = "sha256:ccacf9cb84b64939ea15f859a146af1f662a6b1d68175754a07315e305fb1403"}, @@ -282,10 +268,9 @@ files = [ name = "aliyun-python-sdk-core" version = "2.11.5" description = "The core module of Aliyun Python SDK." -optional = true +optional = false python-versions = "*" groups = ["main"] -markers = "extra == \"eval\"" files = [ {file = "aliyun-python-sdk-core-2.11.5.tar.gz", hash = "sha256:577265c630c02207c692ca19958bd21665d56208306a834d0885e7770553975e"}, ] @@ -297,10 +282,9 @@ pycryptodome = ">=3.4.7" name = "aliyun-python-sdk-kms" version = "2.16.5" description = "The kms module of Aliyun Python sdk." -optional = true +optional = false python-versions = "*" groups = ["main"] -markers = "extra == \"eval\"" files = [ {file = "aliyun-python-sdk-kms-2.16.5.tar.gz", hash = "sha256:f328a8a19d83ecbb965ffce0ec1e9930755216d104638cd95ecd362753b813b3"}, {file = "aliyun_python_sdk_kms-2.16.5-py2.py3-none-any.whl", hash = "sha256:24b6cdc4fd161d2942619479c8d050c63ea9cd22b044fe33b60bbb60153786f0"}, @@ -340,7 +324,7 @@ description = "The official Python library for the anthropic API" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "anthropic-0.84.0-py3-none-any.whl", hash = "sha256:861c4c50f91ca45f942e091d83b60530ad6d4f98733bfe648065364da05d29e7"}, {file = "anthropic-0.84.0.tar.gz", hash = "sha256:72f5f90e5aebe62dca316cb013629cfa24996b0f5a4593b8c3d712bc03c43c37"}, @@ -386,7 +370,6 @@ files = [ ] [package.dependencies] -exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} @@ -400,7 +383,7 @@ description = "tvm ffi" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "apache_tvm_ffi-0.1.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d911cbbc83bf12a0d9ec03e5315ff1bb92d95702fe912cd7a050393274382e71"}, {file = "apache_tvm_ffi-0.1.9-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1c8dd4018420c0d14bace688594710909ce198056ff8ac2ad1cd462b30fe1bdd"}, @@ -454,25 +437,12 @@ description = "Read/rewrite/write Python ASTs" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5"}, {file = "astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e"}, ] -[[package]] -name = "async-timeout" -version = "5.0.1" -description = "Timeout context manager for asyncio programs" -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "python_full_version < \"3.11.3\" and extra == \"server\" or python_version == \"3.10\"" -files = [ - {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, - {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, -] - [[package]] name = "attrs" version = "25.4.0" @@ -501,19 +471,6 @@ files = [ [package.extras] dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] -[[package]] -name = "backports-asyncio-runner" -version = "1.2.0" -description = "Backport of asyncio.Runner, a context manager that controls event loop life cycle." -optional = true -python-versions = "<3.11,>=3.8" -groups = ["main"] -markers = "python_version == \"3.10\" and extra == \"test\"" -files = [ - {file = "backports_asyncio_runner-1.2.0-py3-none-any.whl", hash = "sha256:0da0a936a8aeb554eccb426dc55af3ba63bcdc69fa1a600b5bb305413a4477b5"}, - {file = "backports_asyncio_runner-1.2.0.tar.gz", hash = "sha256:a5aa7b2b7d8f8bfcaa2b57313f70792df84e32a2a746f585213373f900b42162"}, -] - [[package]] name = "beautifulsoup4" version = "4.14.3" @@ -545,7 +502,7 @@ description = "Python bindings for the Rust blake3 crate" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "blake3-1.0.8-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:8956bb9aec47b6c37ccce935a943588f1f5e6e2e85d43bb7cb76a574238f8a9b"}, {file = "blake3-1.0.8-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7adbbee5dd0c302218eb8acdfd82b7006930eb5798f56f79f9cca89f6f192662"}, @@ -668,7 +625,7 @@ description = "Extensible memoizing collections and decorators" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "cachetools-7.0.1-py3-none-any.whl", hash = "sha256:8f086515c254d5664ae2146d14fc7f65c9a4bce75152eb247e5a9c5e6d7b2ecf"}, {file = "cachetools-7.0.1.tar.gz", hash = "sha256:e31e579d2c5b6e2944177a0397150d312888ddf4e16e12f1016068f0c03b8341"}, @@ -681,7 +638,7 @@ description = "CBOR (de)serializer with extensive tag support" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "cbor2-5.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2263c0c892194f10012ced24c322d025d9d7b11b41da1c357f3b3fe06676e6b7"}, {file = "cbor2-5.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ffe4ca079f6f8ed393f5c71a8de22651cb27bd50e74e2bcd6bc9c8f853a732b"}, @@ -745,10 +702,10 @@ files = [ name = "cffi" version = "2.0.0" description = "Foreign Function Interface for Python calling C code." -optional = false +optional = true python-versions = ">=3.9" groups = ["main"] -markers = "(platform_python_implementation != \"PyPy\" or implementation_name == \"pypy\") and (implementation_name == \"pypy\" or extra == \"rl\" or extra == \"async-rl\")" +markers = "platform_python_implementation != \"PyPy\" and (extra == \"vllm\" or extra == \"ray\") or extra == \"vllm\" and implementation_name == \"pypy\"" files = [ {file = "cffi-2.0.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:0cf2d91ecc3fcc0625c2c530fe004f82c110405f101548512cce44322fa8ac44"}, {file = "cffi-2.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f73b96c41e3b2adedc34a7356e64c8eb96e03a3782b535e043a986276ce12a49"}, @@ -966,9 +923,10 @@ files = [ name = "click" version = "8.3.1" description = "Composable command line interface toolkit" -optional = false +optional = true python-versions = ">=3.10" groups = ["main"] +markers = "extra == \"vllm\" or extra == \"ray\" or extra == \"tinker\"" files = [ {file = "click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6"}, {file = "click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a"}, @@ -984,7 +942,7 @@ description = "Pickler class to extend the standard pickle.Pickler functionality optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a"}, {file = "cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414"}, @@ -997,7 +955,7 @@ description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" groups = ["main"] -markers = "(platform_system == \"Windows\" or sys_platform == \"win32\" or extra == \"eval\") and (platform_system == \"Windows\" or extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"test\" or extra == \"docs\")" +markers = "(extra == \"vllm\" or extra == \"ray\" or extra == \"docs\") and sys_platform == \"win32\" or platform_system == \"Windows\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -1010,7 +968,7 @@ description = "Terminal string styling done right, in Python." optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "colorful-0.5.8-py2.py3-none-any.whl", hash = "sha256:a9381fdda3337fbaba5771991020abc69676afa102646650b759927892875992"}, {file = "colorful-0.5.8.tar.gz", hash = "sha256:bb16502b198be2f1c42ba3c52c703d5f651d826076817185f0294c1a549a7445"}, @@ -1019,25 +977,6 @@ files = [ [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -[[package]] -name = "colorlog" -version = "6.12.0" -description = "Add colours to the output of Python's logging module." -optional = true -python-versions = ">=3.6" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "colorlog-6.12.0-py3-none-any.whl", hash = "sha256:30d392604e9110045a2c2aeefc27d7a017abbab63f3a8aee594eac0801df784e"}, - {file = "colorlog-6.12.0.tar.gz", hash = "sha256:2a7924c1dadf18b22a0eb8b06d1c7b01d5341707ec1641eb6fcc4fde0c3e8e5f"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} - -[package.extras] -development = ["black", "flake8", "mypy", "pytest", "types-colorama"] - [[package]] name = "commonmark" version = "0.9.1" @@ -1061,7 +1000,7 @@ description = "Library for utilization of compressed safetensors of neural netwo optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "compressed_tensors-0.13.0-py3-none-any.whl", hash = "sha256:3518799c9baf034eb642efb551db6b0537b8713d45a64fe4def26f7f8d6cabec"}, {file = "compressed_tensors-0.13.0.tar.gz", hash = "sha256:23893824d3498ea3f1a829f14a8fa85f9a5e76a34c711a038b8d7c619ca9a67c"}, @@ -1081,10 +1020,9 @@ dev = ["black (==22.12.0)", "flake8 (>=3.8.3)", "isort (==5.8.0)", "nbconvert (> name = "crcmod" version = "1.7" description = "CRC Generator" -optional = true +optional = false python-versions = "*" groups = ["main"] -markers = "extra == \"eval\"" files = [ {file = "crcmod-1.7.tar.gz", hash = "sha256:dc7051a0db5f2bd48665a990d3ec1cc305a466a77358ca4492826f41f283601e"}, ] @@ -1096,7 +1034,7 @@ description = "cryptography is a package which provides cryptographic recipes an optional = true python-versions = "!=3.9.0,!=3.9.1,>=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\" or extra == \"vllm\"" files = [ {file = "cryptography-46.0.5-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:351695ada9ea9618b3500b490ad54c739860883df6c1f555e088eaf25b1bbaad"}, {file = "cryptography-46.0.5-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c18ff11e86df2e28854939acde2d003f7984f721eba450b56a200ad90eeb0e6b"}, @@ -1151,7 +1089,6 @@ files = [ [package.dependencies] cffi = {version = ">=2.0.0", markers = "python_full_version >= \"3.9.0\" and platform_python_implementation != \"PyPy\""} -typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11.0\""} [package.extras] docs = ["sphinx (>=5.3.0)", "sphinx-inline-tabs", "sphinx-rtd-theme (>=3.0.0)"] @@ -1170,7 +1107,7 @@ description = "Python bindings for CUDA" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "cuda_bindings-13.1.1-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4400370a83f1538e25ed4c18c34a0e9d5fad39741e282e69ce24d1479a11017d"}, {file = "cuda_bindings-13.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:81f92500e2f6aec2dac00a5a1ce77d5aa77ea77b606dc484d951f1f2cc3eaa13"}, @@ -1205,7 +1142,7 @@ description = "Pathfinder for CUDA components" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "cuda_pathfinder-1.4.0-py3-none-any.whl", hash = "sha256:437079ca59e7b61ae439ecc501d69ed87b3accc34d58153ef1e54815e2c2e118"}, ] @@ -1217,7 +1154,7 @@ description = "CUDA Python: Performance meets Productivity" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "cuda_python-13.1.1-py3-none-any.whl", hash = "sha256:944cc4fe6482673d28dd545797a28840945a1668739328fa2ad1e9be4f7050d9"}, ] @@ -1236,7 +1173,7 @@ description = "CuPy: NumPy & SciPy for GPU" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "sys_platform != \"darwin\" and (extra == \"rl\" or extra == \"async-rl\")" +markers = "extra == \"vllm\" and sys_platform != \"darwin\"" files = [ {file = "cupy_cuda12x-14.0.1-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:b42da54c9da0d5a7748e4120f13c47594d3e1fc2741b712591aa915517741096"}, {file = "cupy_cuda12x-14.0.1-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:7c775e1e1ebc0c4c9f94a4c6bb66a0c07d109de5dfcef671f9e4056df4bd81ca"}, @@ -1307,19 +1244,6 @@ tests-numpy2 = ["Pillow (>=9.4.0)", "absl-py", "aiohttp", "decorator", "elastics torch = ["torch"] vision = ["Pillow (>=9.4.0)"] -[[package]] -name = "defusedxml" -version = "0.7.1" -description = "XML bomb protection for Python stdlib modules" -optional = true -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, - {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, -] - [[package]] name = "depyf" version = "0.20.0" @@ -1327,7 +1251,7 @@ description = "Decompile python functions, from bytecode to source code!" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "depyf-0.20.0-py3-none-any.whl", hash = "sha256:d31effad4261cebecb58955d832e448ace88f432328f95f82fd99c30fd9308d4"}, {file = "depyf-0.20.0.tar.gz", hash = "sha256:fb7683bd72c44f67b56029df2c47721e9a02ffa4d7b19095f1c54c4ebf797a98"}, @@ -1363,7 +1287,7 @@ description = "Disk Cache -- Disk and file backed persistent cache." optional = true python-versions = ">=3" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19"}, {file = "diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc"}, @@ -1376,7 +1300,7 @@ description = "Distribution utilities" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, @@ -1389,7 +1313,7 @@ description = "Distro - an OS platform information API" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" +markers = "extra == \"vllm\" or extra == \"tinker\"" files = [ {file = "distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2"}, {file = "distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed"}, @@ -1402,7 +1326,7 @@ description = "DNS toolkit" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af"}, {file = "dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f"}, @@ -1424,7 +1348,7 @@ description = "Parse Python docstrings in reST, Google and Numpydoc format" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "docstring_parser-0.17.0-py3-none-any.whl", hash = "sha256:cf2569abd23dce8099b300f9b4fa8191e9582dda731fd533daf54c4551658708"}, {file = "docstring_parser-0.17.0.tar.gz", hash = "sha256:583de4a309722b3315439bb31d64ba3eebada841f2e2cee23b99df001434c912"}, @@ -1448,103 +1372,6 @@ files = [ {file = "docutils-0.16.tar.gz", hash = "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc"}, ] -[[package]] -name = "dotenv" -version = "0.9.9" -description = "Deprecated package" -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "dotenv-0.9.9-py2.py3-none-any.whl", hash = "sha256:29cf74a087b31dafdb5a446b6d7e11cbce8ed2741540e2339c69fbef92c94ce9"}, -] - -[package.dependencies] -python-dotenv = "*" - -[[package]] -name = "editdistance" -version = "0.8.1" -description = "Fast implementation of the edit distance (Levenshtein distance)" -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "editdistance-0.8.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:adeb705f32b93accc74960d227875abff150ee42d676e428536361fe5f8f5388"}, - {file = "editdistance-0.8.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3de77951b105d0972deec7684a0b3d1a9dee69c9b5d34f6e2acc0d76cd4a1c52"}, - {file = "editdistance-0.8.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5e88efb052d45e924606c305cb833a80579dca3e8e4ff01309d50ba2c1c0bbd5"}, - {file = "editdistance-0.8.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0247e7a1e9c66ea75211a97e725366bff19a52aac2c838ed5f90025630e976dd"}, - {file = "editdistance-0.8.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67d143429a49ab552411505f550a0fb4285a1d4336e096804d233ec495ac20fc"}, - {file = "editdistance-0.8.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca9d3be2b10e5d44a950a4bd1e84bca9ebbecd364bce0cf5693bf8224c78eaef"}, - {file = "editdistance-0.8.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:5c72aa1df8535f2e2b3d8773a1a7da091bc1a7e52bb396e7e48d375ba687e7b2"}, - {file = "editdistance-0.8.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:9a606c34a2a6cc190e4fffc856b36333cdcf1f1fab5b22bd3088e585c22d6ca0"}, - {file = "editdistance-0.8.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5af173d442ffac33b7c7990132f97f88818a3abf4b21c0c702a7022df37c0c5c"}, - {file = "editdistance-0.8.1-cp310-cp310-win32.whl", hash = "sha256:fd64b58f5a7b59afd9d75982aaeeacd2a98498bf472fa0360c122ffe6ea4c871"}, - {file = "editdistance-0.8.1-cp310-cp310-win_amd64.whl", hash = "sha256:6c7c62c3cae45ca1fa01bb2722b297b9de1e3a244ac44cfba88bdcb488fe6aee"}, - {file = "editdistance-0.8.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:486105603a273d73d12a54f347dffa70ab281749d7c3879658b377bc49e4b98c"}, - {file = "editdistance-0.8.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:fad081f5f86a175c1a09a4e9e45b95c9349e454c21e181e842e01c85f1f536fc"}, - {file = "editdistance-0.8.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8cb78e125f6759398885a775f5eed07c2bb72b2f86da43e674c6b6a3335b273b"}, - {file = "editdistance-0.8.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3778ca60aa89def9144b70e330bcec5330c7da1d69cb28c612e90b84510a1d3d"}, - {file = "editdistance-0.8.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fba945eaa0436cf40bc53d7e299dc537c7c71353379a095b7459ff4af910da33"}, - {file = "editdistance-0.8.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:877f2a0d801f32bc1a1878901ffb947b974361e849c66e314a7f1d786a446b58"}, - {file = "editdistance-0.8.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e79d351ca40a6ead5f3763253fd7521572ee0d3e5d42538630e56d10f48db481"}, - {file = "editdistance-0.8.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:70ed382b3052a51161bad0149d4665003bf3b949fce0b01bf1253a4cc1a88239"}, - {file = "editdistance-0.8.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a529bfb384c4000775d76739c4e64f73337f0f5a3784933b1321b577a62bed4e"}, - {file = "editdistance-0.8.1-cp311-cp311-win32.whl", hash = "sha256:b082232429e731f181af7f7d2bcf79da6ca8fadd04e9086c11e2973f7d330c81"}, - {file = "editdistance-0.8.1-cp311-cp311-win_amd64.whl", hash = "sha256:cef1a4359252a49f2c4718e64e9d40027d9d951b289d045bdb278656e59f6af8"}, - {file = "editdistance-0.8.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b04af61b3fcdd287a07c15b6ae3b02af01c5e3e9c3aca76b8c1d13bd266b6f57"}, - {file = "editdistance-0.8.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:18fc8b6eaae01bfd9cf999af726c1e8dcf667d120e81aa7dbd515bea7427f62f"}, - {file = "editdistance-0.8.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6a87839450a5987028738d061ffa5ef6a68bac2ddc68c9147a8aae9806629c7f"}, - {file = "editdistance-0.8.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24b5f9c9673c823d91b5973d0af8b39f883f414a55ade2b9d097138acd10f31e"}, - {file = "editdistance-0.8.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c59248eabfad603f0fba47b0c263d5dc728fb01c2b6b50fb6ca187cec547fdb3"}, - {file = "editdistance-0.8.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e239d88ff52821cf64023fabd06a1d9a07654f364b64bf1284577fd3a79d0e"}, - {file = "editdistance-0.8.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2f7f71698f83e8c83839ac0d876a0f4ef996c86c5460aebd26d85568d4afd0db"}, - {file = "editdistance-0.8.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:04e229d6f4ce0c12abc9f4cd4023a5b5fa9620226e0207b119c3c2778b036250"}, - {file = "editdistance-0.8.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:e16721636da6d6b68a2c09eaced35a94f4a4a704ec09f45756d4fd5e128ed18d"}, - {file = "editdistance-0.8.1-cp312-cp312-win32.whl", hash = "sha256:87533cf2ebc3777088d991947274cd7e1014b9c861a8aa65257bcdc0ee492526"}, - {file = "editdistance-0.8.1-cp312-cp312-win_amd64.whl", hash = "sha256:09f01ed51746d90178af7dd7ea4ebb41497ef19f53c7f327e864421743dffb0a"}, - {file = "editdistance-0.8.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0b6f52a9d7d434f6882db3dc0340e42da6f177644c23f6a02a739b6247a14b82"}, - {file = "editdistance-0.8.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d5b413aeb8fe3f77a9e95485fcb64ce84e6b14bcd368124d38bd0062167b3456"}, - {file = "editdistance-0.8.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4ba571e8b6796ad34faeb8581ddc311c35946b2fc183eaebfef59e12ea3538b3"}, - {file = "editdistance-0.8.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e361e932a85ee1f7091fcb8f7e4619681592c1a0cca251dfd26976dd58254"}, - {file = "editdistance-0.8.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf200104ed4923d4f51ca543bc8488732a31a17848058e65bcba855a7eee2bd2"}, - {file = "editdistance-0.8.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:387d3bb45befbf8514eb8d17180307580efe4ebaa40ad8b2c14eb04c52ad18a0"}, - {file = "editdistance-0.8.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8e4ea90e92f0e3494bdddad45928393094c258aeef9e4def81a39c3429df0e19"}, - {file = "editdistance-0.8.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:5d746ecbf7db7fe0f93ba6971ac43225aac877818a3c2003d41436fee9b33905"}, - {file = "editdistance-0.8.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d789a8ef6fe7cb287ff199381bbd62e9fb2ba4b1e69db817b761d9b42bf8ca7b"}, - {file = "editdistance-0.8.1-cp38-cp38-win32.whl", hash = "sha256:dbbf050fece6c78838a8a95fa4e9a4132023c3d85138870ac83c9dc1dfbfe513"}, - {file = "editdistance-0.8.1-cp38-cp38-win_amd64.whl", hash = "sha256:1c49df0717f64a2c8869edc32c01ba9a22ba20b6cc482876c1067e3a92a6cb2d"}, - {file = "editdistance-0.8.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4d8e9a3e65a68c13dcadc1d2caca620f1716a8d02f2602047e0721b509161ec7"}, - {file = "editdistance-0.8.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7994a6a0a6ae92db87c144e12f1549ca0e50f43c6cc64e32c628e7af6b9c74b6"}, - {file = "editdistance-0.8.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:dbe0cbc15466e9b7fbf73e34bdcae11cb0c2acd09a60ef4740f2172f9aa5e751"}, - {file = "editdistance-0.8.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bc5f0c7f12a3a3bf2d129e2900deaaa5e47203ef61918343ddc4b6c03e50f089"}, - {file = "editdistance-0.8.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:98572c662fd7d425ff24acb8197ad4be7849558a48aebbc60012090bfda4dce9"}, - {file = "editdistance-0.8.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b35c647a8a17b77441e7b6111b74ae1016851589109e1efc990d27225b3217b"}, - {file = "editdistance-0.8.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2f56c0e006f6b5207985c1bdd62e1873e66bb06a60849cad32716cad1bb3ae40"}, - {file = "editdistance-0.8.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d6bc5a827b262dc9b0d03cfd821682334ce1280520edf6385dc1730e390b5201"}, - {file = "editdistance-0.8.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ad68a2357664e45823b38c9f67a315ff9771263ec502a710057b78c6ca6fcfcd"}, - {file = "editdistance-0.8.1-cp39-cp39-win32.whl", hash = "sha256:16b3e413c020e42b2ef2d4ba01386ead43007217f0bdd704e90474ace90d2023"}, - {file = "editdistance-0.8.1-cp39-cp39-win_amd64.whl", hash = "sha256:331f1a8d3a753858a9d689c0bcd79ad1959e0df464bb6c22cb263cfb6da208e4"}, - {file = "editdistance-0.8.1-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a4a90c6b03094c07358572027a8d0a13cca7450b1aa6caca98a5f1fa4f0b8961"}, - {file = "editdistance-0.8.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:510a4f9ced348a4fd89ae2e102357d4d801a771e29bb2bc2f130a1692193407f"}, - {file = "editdistance-0.8.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4787fa7228ba6a34b430066d174320f011d605015baa7299c2c4911e6ea6bd46"}, - {file = "editdistance-0.8.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee02601375073afccd6b4d811129ce1cb696d47db734784d8dbd1fddcea75447"}, - {file = "editdistance-0.8.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:bc7ad9f9a20e6f351523de77c59249f005242e3f317b5de45d02c378d24f6531"}, - {file = "editdistance-0.8.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7743895df48482fa5a7136543d6bde72e6c10c78a4a4b772fcddda48f792ef68"}, - {file = "editdistance-0.8.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d9f139d921aff07deb2c9e592fe23d994af0e59267962a20c062cd66750a0ca4"}, - {file = "editdistance-0.8.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:493587829de3e500bdf34f03f5ab12501867b911acc838e1d04047a3f8941aad"}, - {file = "editdistance-0.8.1-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:16c521157f5d29bf2cb20472482de8450722685d27a6dd801ff1e80cb13a1fd1"}, - {file = "editdistance-0.8.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:2a956eb9584d9e8d165bddd9091791924648071c3cdb1e03ec94b1320c2edefd"}, - {file = "editdistance-0.8.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:8131acb6b5170382b8b74efab92df8739ac591dc841314e0153af63c4493cb43"}, - {file = "editdistance-0.8.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f182e5e1d2a446138cab085409395c62af36eb1abcbe8cfacb083febfeafd5ce"}, - {file = "editdistance-0.8.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f4f19a829aff230377041acb77afec73becbebafe35b7e322be00cdb3122ddb"}, - {file = "editdistance-0.8.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b978c5927100a57791131dd2418040f4e5d33970d37b97a84c1a530ec481f557"}, - {file = "editdistance-0.8.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:0c96a8e981f385f0b7392d047c5caab8e0b24f94b71120787fd78241efc34237"}, - {file = "editdistance-0.8.1.tar.gz", hash = "sha256:d1cdf80a5d5014b0c9126a69a42ce55a457b457f6986ff69ca98e4fe4d2d8fed"}, -] - [[package]] name = "einops" version = "0.8.2" @@ -1564,7 +1391,7 @@ description = "A robust email address syntax and deliverability validation libra optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4"}, {file = "email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426"}, @@ -1574,115 +1401,6 @@ files = [ dnspython = ">=2.0.0" idna = ">=2.0.0" -[[package]] -name = "evalscope" -version = "1.11.1" -description = "EvalScope: Lightweight LLMs Evaluation Framework" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "evalscope-1.11.1-py3-none-any.whl", hash = "sha256:5058c5112ee0dfff0048a48ee1ff9c6e1ca28bbed39ef5cd34a6a14d3ddf2b66"}, - {file = "evalscope-1.11.1.tar.gz", hash = "sha256:4a6f4437a37543a608a08f5279eb215a2832d32d1ecc812ba8442a67c3340460"}, -] - -[package.dependencies] -aiohttp = "*" -colorlog = "*" -docstring_parser = "*" -dotenv = "*" -editdistance = "*" -filetype = "*" -jieba = "*" -jinja2 = "*" -jsonlines = "*" -jsonschema = "*" -latex2sympy2_extended = {version = "*", extras = ["antlr4-9-3"]} -litellm = ">=1.55,<2.0" -Markdown = "*" -modelscope = {version = ">=1.34", extras = ["datasets"]} -more_itertools = "*" -nltk = "*" -openai = "*" -overrides = "*" -pandas = "*" -pillow = "*" -plotly = "*" -pydantic = "*" -pylatexenc = "*" -pyyaml = ">=5.1" -requests = "*" -rich = "*" -rouge-chinese = "*" -rouge-score = ">=0.1.0" -sacrebleu = "*" -sympy = "*" -tabulate = "*" -tqdm = "*" -transformers = ">=4.33,<4.57.2 || >4.57.2" -word2number = "*" -zhconv = "*" - -[package.extras] -aigc = ["diffusers", "iopath", "lpips", "omegaconf", "open_clip_torch", "opencv-python", "peft (>=0.17)", "torch", "torchvision"] -air-bench = ["soundfile"] -all = ["aiohttp", "diffusers", "fastapi", "flask (>=2.0.0)", "flask (>=2.0.0)", "instructor", "iopath", "jinja2", "langchain (>=0.3.0)", "langchain-community (>=0.3.0)", "langchain-core (>=0.3.0)", "langchain-huggingface (>=0.1.0)", "langchain-openai (>=0.3.0)", "lpips", "modelscope[datasets] (>=1.10.0)", "ms-opencompass (>=0.1.6)", "ms-vlmeval (>=0.0.17)", "mteb (>=2.7.0,<3.0.0)", "numpy", "omegaconf", "open_clip_torch", "openai (>=1.0.0)", "opencv-python", "peft (>=0.17)", "plotly (>=5.23.0)", "plotly (>=5.23.0)", "ragas (>=0.4.0,<0.5.0)", "rich", "sentence-transformers (>=5.4.0)", "sse_starlette", "torch", "torch (>=2.0.0)", "torchvision", "transformers", "unstructured", "uvicorn", "uvloop ; sys_platform != \"win32\"", "webdataset (>0.2.0)"] -app = ["flask (>=2.0.0)", "plotly (>=5.23.0)"] -arena-hard = ["scikit-learn"] -bfcl = ["bfcl-eval (==2025.10.27.1)"] -caption = ["pycocoevalcap"] -cc-ocr-v2 = ["apted", "distance", "lxml", "python-Levenshtein", "scipy", "zss"] -deep-swe = ["datacurve-pier (>=0.3.0)"] -dev = ["hypothesis (==6.155.2)", "pre-commit (==4.6.0)", "pytest", "pytest-cov", "python-dotenv"] -docs = ["docutils (>=0.16.0)", "myst_parser", "recommonmark", "sphinx (>=5.3.0)", "sphinx-design", "sphinxawesome-theme"] -general-arena = ["scikit-learn"] -ifbench = ["emoji", "nltk (>=3.9)", "syllapy"] -ifeval = ["langdetect", "nltk (>=3.9)"] -maritime-ocr-bench = ["numpy", "pillow", "shapely"] -mcp = ["mcp (>=1.0)", "mcp-server-fetch"] -miniwob = ["browsergym-miniwob (==0.14.3)"] -multi-if = ["emoji", "langdetect", "nltk (>=3.9)", "pythainlp"] -needle-haystack = ["matplotlib", "seaborn"] -ocr-bench = ["Polygon3", "apted", "distance", "editdistance", "jieba", "lxml", "nltk (>=3.9)", "python-Levenshtein", "zss"] -olmocr-bench = ["beautifulsoup4", "fuzzysearch", "rapidfuzz"] -olympiad-bench = ["latex2sympy2_extended[antlr4-9-3]"] -omnidoc-bench = ["apted", "beautifulsoup4", "distance", "jieba", "lxml", "python-Levenshtein"] -openai-mrcr = ["tiktoken"] -opencompass = ["ms-opencompass (>=0.1.6)"] -perf = ["aiohttp", "fastapi", "jinja2", "numpy", "rich", "sse_starlette", "transformers", "uvicorn", "uvloop ; sys_platform != \"win32\""] -rag = ["instructor", "langchain (>=0.3.0)", "langchain-community (>=0.3.0)", "langchain-core (>=0.3.0)", "langchain-huggingface (>=0.1.0)", "langchain-openai (>=0.3.0)", "modelscope[datasets] (>=1.10.0)", "mteb (>=2.7.0,<3.0.0)", "openai (>=1.0.0)", "ragas (>=0.4.0,<0.5.0)", "sentence-transformers (>=5.4.0)", "torch (>=2.0.0)", "unstructured", "webdataset (>0.2.0)"] -refcoco = ["pycocoevalcap"] -sandbox = ["ms-enclave[docker] (>=0.0.8)"] -service = ["aiohttp", "fastapi", "flask (>=2.0.0)", "jinja2", "numpy", "plotly (>=5.23.0)", "rich", "sse_starlette", "transformers", "uvicorn", "uvloop ; sys_platform != \"win32\""] -swe-bench = ["swebench (==4.1.0)"] -terminal-bench = ["harbor (>=0.8.0,<1.0.0)"] -toolathlon = ["httpx", "websockets"] -torgo = ["jellyfish", "jiwer"] -vlmeval = ["ms-vlmeval (>=0.0.17)"] -vtcbench = ["beautifulsoup4"] -wide-search = ["dateparser"] -wmt = ["unbabel-comet"] - -[[package]] -name = "exceptiongroup" -version = "1.3.1" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598"}, - {file = "exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219"}, -] - -[package.dependencies] -typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} - -[package.extras] -test = ["pytest (>=6)"] - [[package]] name = "fastapi" version = "0.135.1" @@ -1722,7 +1440,7 @@ description = "Run and manage FastAPI apps from the command line with FastAPI CL optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "fastapi_cli-0.0.24-py3-none-any.whl", hash = "sha256:4a1f78ed798f106b4fee85ca93b85d8fe33c0a3570f775964d37edb80b8f0edc"}, {file = "fastapi_cli-0.0.24.tar.gz", hash = "sha256:1afc9c9e21d7ebc8a3ca5e31790cd8d837742be7e4f8b9236e99cb3451f0de00"}, @@ -1731,7 +1449,6 @@ files = [ [package.dependencies] fastapi-cloud-cli = {version = ">=0.1.1", optional = true, markers = "extra == \"standard\""} rich-toolkit = ">=0.14.8" -tomli = {version = ">=2.0.0", markers = "python_version < \"3.11\""} typer = ">=0.16.0" uvicorn = {version = ">=0.15.0", extras = ["standard"]} @@ -1747,7 +1464,7 @@ description = "Deploy and manage FastAPI Cloud apps from the command line 🚀" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "fastapi_cloud_cli-0.14.0-py3-none-any.whl", hash = "sha256:325fcb4b45e661184152da6db861d9fb718739fbcd561a4d334dbe78c026586f"}, {file = "fastapi_cloud_cli-0.14.0.tar.gz", hash = "sha256:d3ecb8c942685a71df0af7bd59f463b5eff76f5818b48e5a03c6159726831e68"}, @@ -1756,10 +1473,7 @@ files = [ [package.dependencies] fastar = ">=0.8.0" httpx = ">=0.27.0" -pydantic = [ - {version = ">=2.7.4", extras = ["email"], markers = "python_version < \"3.13\""}, - {version = ">=2.8.0", extras = ["email"], markers = "python_version == \"3.13\""}, -] +pydantic = {version = ">=2.7.4", extras = ["email"], markers = "python_version < \"3.13\""} rich-toolkit = ">=0.19.7" rignore = ">=0.5.1" sentry-sdk = ">=2.20.0" @@ -1776,7 +1490,7 @@ description = "High-level bindings for the Rust tar crate" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "fastar-0.8.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c9f930cff014cf79d396d0541bd9f3a3f170c9b5e45d10d634d98f9ed08788c3"}, {file = "fastar-0.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:07b70f712d20622346531a4b46bb332569bea621f61314c0b7e80903a16d14cf"}, @@ -1934,95 +1648,6 @@ files = [ {file = "fastar-0.8.0.tar.gz", hash = "sha256:f4d4d68dbf1c4c2808f0e730fac5843493fc849f70fe3ad3af60dfbaf68b9a12"}, ] -[[package]] -name = "fastuuid" -version = "0.14.0" -description = "Python bindings to Rust's UUID library." -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "fastuuid-0.14.0-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:6e6243d40f6c793c3e2ee14c13769e341b90be5ef0c23c82fa6515a96145181a"}, - {file = "fastuuid-0.14.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:13ec4f2c3b04271f62be2e1ce7e95ad2dd1cf97e94503a3760db739afbd48f00"}, - {file = "fastuuid-0.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b2fdd48b5e4236df145a149d7125badb28e0a383372add3fbaac9a6b7a394470"}, - {file = "fastuuid-0.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f74631b8322d2780ebcf2d2d75d58045c3e9378625ec51865fe0b5620800c39d"}, - {file = "fastuuid-0.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:83cffc144dc93eb604b87b179837f2ce2af44871a7b323f2bfed40e8acb40ba8"}, - {file = "fastuuid-0.14.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a771f135ab4523eb786e95493803942a5d1fc1610915f131b363f55af53b219"}, - {file = "fastuuid-0.14.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:4edc56b877d960b4eda2c4232f953a61490c3134da94f3c28af129fb9c62a4f6"}, - {file = "fastuuid-0.14.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:bcc96ee819c282e7c09b2eed2b9bd13084e3b749fdb2faf58c318d498df2efbe"}, - {file = "fastuuid-0.14.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7a3c0bca61eacc1843ea97b288d6789fbad7400d16db24e36a66c28c268cfe3d"}, - {file = "fastuuid-0.14.0-cp310-cp310-win32.whl", hash = "sha256:7f2f3efade4937fae4e77efae1af571902263de7b78a0aee1a1653795a093b2a"}, - {file = "fastuuid-0.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:ae64ba730d179f439b0736208b4c279b8bc9c089b102aec23f86512ea458c8a4"}, - {file = "fastuuid-0.14.0-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:73946cb950c8caf65127d4e9a325e2b6be0442a224fd51ba3b6ac44e1912ce34"}, - {file = "fastuuid-0.14.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:12ac85024637586a5b69645e7ed986f7535106ed3013640a393a03e461740cb7"}, - {file = "fastuuid-0.14.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:05a8dde1f395e0c9b4be515b7a521403d1e8349443e7641761af07c7ad1624b1"}, - {file = "fastuuid-0.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09378a05020e3e4883dfdab438926f31fea15fd17604908f3d39cbeb22a0b4dc"}, - {file = "fastuuid-0.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbb0c4b15d66b435d2538f3827f05e44e2baafcc003dd7d8472dc67807ab8fd8"}, - {file = "fastuuid-0.14.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cd5a7f648d4365b41dbf0e38fe8da4884e57bed4e77c83598e076ac0c93995e7"}, - {file = "fastuuid-0.14.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c0a94245afae4d7af8c43b3159d5e3934c53f47140be0be624b96acd672ceb73"}, - {file = "fastuuid-0.14.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:2b29e23c97e77c3a9514d70ce343571e469098ac7f5a269320a0f0b3e193ab36"}, - {file = "fastuuid-0.14.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1e690d48f923c253f28151b3a6b4e335f2b06bf669c68a02665bc150b7839e94"}, - {file = "fastuuid-0.14.0-cp311-cp311-win32.whl", hash = "sha256:a6f46790d59ab38c6aa0e35c681c0484b50dc0acf9e2679c005d61e019313c24"}, - {file = "fastuuid-0.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:e150eab56c95dc9e3fefc234a0eedb342fac433dacc273cd4d150a5b0871e1fa"}, - {file = "fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77e94728324b63660ebf8adb27055e92d2e4611645bf12ed9d88d30486471d0a"}, - {file = "fastuuid-0.14.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:caa1f14d2102cb8d353096bc6ef6c13b2c81f347e6ab9d6fbd48b9dea41c153d"}, - {file = "fastuuid-0.14.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d23ef06f9e67163be38cece704170486715b177f6baae338110983f99a72c070"}, - {file = "fastuuid-0.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c9ec605ace243b6dbe3bd27ebdd5d33b00d8d1d3f580b39fdd15cd96fd71796"}, - {file = "fastuuid-0.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:808527f2407f58a76c916d6aa15d58692a4a019fdf8d4c32ac7ff303b7d7af09"}, - {file = "fastuuid-0.14.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fb3c0d7fef6674bbeacdd6dbd386924a7b60b26de849266d1ff6602937675c8"}, - {file = "fastuuid-0.14.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab3f5d36e4393e628a4df337c2c039069344db5f4b9d2a3c9cea48284f1dd741"}, - {file = "fastuuid-0.14.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b9a0ca4f03b7e0b01425281ffd44e99d360e15c895f1907ca105854ed85e2057"}, - {file = "fastuuid-0.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3acdf655684cc09e60fb7e4cf524e8f42ea760031945aa8086c7eae2eeeabeb8"}, - {file = "fastuuid-0.14.0-cp312-cp312-win32.whl", hash = "sha256:9579618be6280700ae36ac42c3efd157049fe4dd40ca49b021280481c78c3176"}, - {file = "fastuuid-0.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:d9e4332dc4ba054434a9594cbfaf7823b57993d7d8e7267831c3e059857cf397"}, - {file = "fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77a09cb7427e7af74c594e409f7731a0cf887221de2f698e1ca0ebf0f3139021"}, - {file = "fastuuid-0.14.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9bd57289daf7b153bfa3e8013446aa144ce5e8c825e9e366d455155ede5ea2dc"}, - {file = "fastuuid-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ac60fc860cdf3c3f327374db87ab8e064c86566ca8c49d2e30df15eda1b0c2d5"}, - {file = "fastuuid-0.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ab32f74bd56565b186f036e33129da77db8be09178cd2f5206a5d4035fb2a23f"}, - {file = "fastuuid-0.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33e678459cf4addaedd9936bbb038e35b3f6b2061330fd8f2f6a1d80414c0f87"}, - {file = "fastuuid-0.14.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1e3cc56742f76cd25ecb98e4b82a25f978ccffba02e4bdce8aba857b6d85d87b"}, - {file = "fastuuid-0.14.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cb9a030f609194b679e1660f7e32733b7a0f332d519c5d5a6a0a580991290022"}, - {file = "fastuuid-0.14.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:09098762aad4f8da3a888eb9ae01c84430c907a297b97166b8abc07b640f2995"}, - {file = "fastuuid-0.14.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1383fff584fa249b16329a059c68ad45d030d5a4b70fb7c73a08d98fd53bcdab"}, - {file = "fastuuid-0.14.0-cp313-cp313-win32.whl", hash = "sha256:a0809f8cc5731c066c909047f9a314d5f536c871a7a22e815cc4967c110ac9ad"}, - {file = "fastuuid-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:0df14e92e7ad3276327631c9e7cec09e32572ce82089c55cb1bb8df71cf394ed"}, - {file = "fastuuid-0.14.0-cp314-cp314-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:b852a870a61cfc26c884af205d502881a2e59cc07076b60ab4a951cc0c94d1ad"}, - {file = "fastuuid-0.14.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c7502d6f54cd08024c3ea9b3514e2d6f190feb2f46e6dbcd3747882264bb5f7b"}, - {file = "fastuuid-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1ca61b592120cf314cfd66e662a5b54a578c5a15b26305e1b8b618a6f22df714"}, - {file = "fastuuid-0.14.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa75b6657ec129d0abded3bec745e6f7ab642e6dba3a5272a68247e85f5f316f"}, - {file = "fastuuid-0.14.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8a0dfea3972200f72d4c7df02c8ac70bad1bb4c58d7e0ec1e6f341679073a7f"}, - {file = "fastuuid-0.14.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1bf539a7a95f35b419f9ad105d5a8a35036df35fdafae48fb2fd2e5f318f0d75"}, - {file = "fastuuid-0.14.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:9a133bf9cc78fdbd1179cb58a59ad0100aa32d8675508150f3658814aeefeaa4"}, - {file = "fastuuid-0.14.0-cp314-cp314-musllinux_1_1_i686.whl", hash = "sha256:f54d5b36c56a2d5e1a31e73b950b28a0d83eb0c37b91d10408875a5a29494bad"}, - {file = "fastuuid-0.14.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:ec27778c6ca3393ef662e2762dba8af13f4ec1aaa32d08d77f71f2a70ae9feb8"}, - {file = "fastuuid-0.14.0-cp314-cp314-win32.whl", hash = "sha256:e23fc6a83f112de4be0cc1990e5b127c27663ae43f866353166f87df58e73d06"}, - {file = "fastuuid-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:df61342889d0f5e7a32f7284e55ef95103f2110fee433c2ae7c2c0956d76ac8a"}, - {file = "fastuuid-0.14.0-cp38-cp38-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:47c821f2dfe95909ead0085d4cb18d5149bca704a2b03e03fb3f81a5202d8cea"}, - {file = "fastuuid-0.14.0-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3964bab460c528692c70ab6b2e469dd7a7b152fbe8c18616c58d34c93a6cf8d4"}, - {file = "fastuuid-0.14.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:c501561e025b7aea3508719c5801c360c711d5218fc4ad5d77bf1c37c1a75779"}, - {file = "fastuuid-0.14.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dce5d0756f046fa792a40763f36accd7e466525c5710d2195a038f93ff96346"}, - {file = "fastuuid-0.14.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:193ca10ff553cf3cc461572da83b5780fc0e3eea28659c16f89ae5202f3958d4"}, - {file = "fastuuid-0.14.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0737606764b29785566f968bd8005eace73d3666bd0862f33a760796e26d1ede"}, - {file = "fastuuid-0.14.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:e0976c0dff7e222513d206e06341503f07423aceb1db0b83ff6851c008ceee06"}, - {file = "fastuuid-0.14.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6fbc49a86173e7f074b1a9ec8cf12ca0d54d8070a85a06ebf0e76c309b84f0d0"}, - {file = "fastuuid-0.14.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:de01280eabcd82f7542828ecd67ebf1551d37203ecdfd7ab1f2e534edb78d505"}, - {file = "fastuuid-0.14.0-cp38-cp38-win32.whl", hash = "sha256:af5967c666b7d6a377098849b07f83462c4fedbafcf8eb8bc8ff05dcbe8aa209"}, - {file = "fastuuid-0.14.0-cp38-cp38-win_amd64.whl", hash = "sha256:c3091e63acf42f56a6f74dc65cfdb6f99bfc79b5913c8a9ac498eb7ca09770a8"}, - {file = "fastuuid-0.14.0-cp39-cp39-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:2ec3d94e13712a133137b2805073b65ecef4a47217d5bac15d8ac62376cefdb4"}, - {file = "fastuuid-0.14.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:139d7ff12bb400b4a0c76be64c28cbe2e2edf60b09826cbfd85f33ed3d0bbe8b"}, - {file = "fastuuid-0.14.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d55b7e96531216fc4f071909e33e35e5bfa47962ae67d9e84b00a04d6e8b7173"}, - {file = "fastuuid-0.14.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0eb25f0fd935e376ac4334927a59e7c823b36062080e2e13acbaf2af15db836"}, - {file = "fastuuid-0.14.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:089c18018fdbdda88a6dafd7d139f8703a1e7c799618e33ea25eb52503d28a11"}, - {file = "fastuuid-0.14.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fc37479517d4d70c08696960fad85494a8a7a0af4e93e9a00af04d74c59f9e3"}, - {file = "fastuuid-0.14.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:73657c9f778aba530bc96a943d30e1a7c80edb8278df77894fe9457540df4f85"}, - {file = "fastuuid-0.14.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d31f8c257046b5617fc6af9c69be066d2412bdef1edaa4bdf6a214cf57806105"}, - {file = "fastuuid-0.14.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5816d41f81782b209843e52fdef757a361b448d782452d96abedc53d545da722"}, - {file = "fastuuid-0.14.0-cp39-cp39-win32.whl", hash = "sha256:448aa6833f7a84bfe37dd47e33df83250f404d591eb83527fa2cac8d1e57d7f3"}, - {file = "fastuuid-0.14.0-cp39-cp39-win_amd64.whl", hash = "sha256:84b0779c5abbdec2a9511d5ffbfcd2e53079bf889824b32be170c0d8ef5fc74c"}, - {file = "fastuuid-0.14.0.tar.gz", hash = "sha256:178947fc2f995b38497a74172adee64fdeb8b7ec18f2a5934d037641ba265d26"}, -] - [[package]] name = "filelock" version = "3.25.0" @@ -2035,19 +1660,6 @@ files = [ {file = "filelock-3.25.0.tar.gz", hash = "sha256:8f00faf3abf9dc730a1ffe9c354ae5c04e079ab7d3a683b7c32da5dd05f26af3"}, ] -[[package]] -name = "filetype" -version = "1.2.0" -description = "Infer file type and MIME type of any file/buffer. No external dependencies." -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25"}, - {file = "filetype-1.2.0.tar.gz", hash = "sha256:66b56cd6474bf41d8c54660347d37afcc3f7d1970648de365c102ef77548aadb"}, -] - [[package]] name = "flashinfer-python" version = "0.6.3" @@ -2055,7 +1667,7 @@ description = "FlashInfer: Kernel Library for LLM Serving" optional = true python-versions = "<4.0,>=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "flashinfer_python-0.6.3-py3-none-any.whl", hash = "sha256:0fe2de934a4b3690c543dafb03f38d7bb4a762431abe8ae4f7292d6fef10c65d"}, {file = "flashinfer_python-0.6.3.tar.gz", hash = "sha256:84a762538247a86bc52ff31d9505d161ce1ec059174c1821c87c3ed1e44670fc"}, @@ -2266,7 +1878,7 @@ description = "Read and write ML models in GGUF for GGML" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "gguf-0.18.0-py3-none-any.whl", hash = "sha256:af93f7ef198a265cbde5fa6a6b3101528bca285903949ab0a3e591cd993a1864"}, {file = "gguf-0.18.0.tar.gz", hash = "sha256:b4659093d5d0dccdb5902a904d54b327f4052879fe5e90946ad5fce9f8018c2e"}, @@ -2288,7 +1900,7 @@ description = "Google API client core library" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "google_api_core-2.30.0-py3-none-any.whl", hash = "sha256:80be49ee937ff9aba0fd79a6eddfde35fe658b9953ab9b79c57dd7061afa8df5"}, {file = "google_api_core-2.30.0.tar.gz", hash = "sha256:02edfa9fab31e17fc0befb5f161b3bf93c9096d99aed584625f38065c511ad9b"}, @@ -2297,10 +1909,7 @@ files = [ [package.dependencies] google-auth = ">=2.14.1,<3.0.0" googleapis-common-protos = ">=1.56.3,<2.0.0" -proto-plus = [ - {version = ">=1.22.3,<2.0.0"}, - {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, -] +proto-plus = ">=1.22.3,<2.0.0" protobuf = ">=4.25.8,<7.0.0" requests = ">=2.20.0,<3.0.0" @@ -2315,7 +1924,7 @@ description = "Google Authentication Library" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "google_auth-2.48.0-py3-none-any.whl", hash = "sha256:2e2a537873d449434252a9632c28bfc268b0adb1e53f9fb62afc5333a975903f"}, {file = "google_auth-2.48.0.tar.gz", hash = "sha256:4f7e706b0cd3208a3d940a19a822c37a476ddba5450156c3e6624a71f7c841ce"}, @@ -2344,7 +1953,7 @@ description = "Common protobufs used in Google APIs" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" +markers = "extra == \"ray\"" files = [ {file = "googleapis_common_protos-1.72.0-py3-none-any.whl", hash = "sha256:4299c5a82d5ae1a9702ada957347726b167f9f8d1fc352477702a1e851ff4038"}, {file = "googleapis_common_protos-1.72.0.tar.gz", hash = "sha256:e55a601c1b32b52d7a3e65f43563e2aa61bcd737998ee672ac9b951cd49319f5"}, @@ -2363,7 +1972,7 @@ description = "HTTP/2-based RPC framework" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"server\" or extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "grpcio-1.78.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:7cc47943d524ee0096f973e1081cb8f4f17a4615f2116882a5f1416e4cfe92b5"}, {file = "grpcio-1.78.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:c3f293fdc675ccba4db5a561048cca627b5e7bd1c8a6973ffedabe7d116e22e2"}, @@ -2441,7 +2050,7 @@ description = "Standard Protobuf Reflection Service for gRPC" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "grpcio_reflection-1.78.0-py3-none-any.whl", hash = "sha256:06fcfde9e6888cdd12e9dd1cf6dc7c440c2e9acf420f696ccbe008672ed05b60"}, {file = "grpcio_reflection-1.78.0.tar.gz", hash = "sha256:e6e60c0b85dbcdf963b4d4d150c0f1d238ba891d805b575c52c0365d07fc0c40"}, @@ -2458,7 +2067,7 @@ description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" +markers = "extra == \"vllm\" or extra == \"ray\" or extra == \"tinker\"" files = [ {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, @@ -2471,7 +2080,7 @@ description = "Pure-Python HTTP/2 protocol implementation" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "python_version >= \"3.11\" and extra == \"client\"" +markers = "extra == \"tinker\"" files = [ {file = "h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd"}, {file = "h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1"}, @@ -2527,7 +2136,7 @@ description = "Pure-Python HPACK header encoding" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "python_version >= \"3.11\" and extra == \"client\"" +markers = "extra == \"tinker\"" files = [ {file = "hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496"}, {file = "hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca"}, @@ -2540,7 +2149,7 @@ description = "A minimal low-level HTTP client." optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" +markers = "extra == \"vllm\" or extra == \"tinker\"" files = [ {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, @@ -2563,7 +2172,7 @@ description = "A collection of framework independent HTTP protocol utils." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "httptools-0.7.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:11d01b0ff1fe02c4c32d60af61a4d613b74fad069e47e06e9067758c01e9ac78"}, {file = "httptools-0.7.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:84d86c1e5afdc479a6fdabf570be0d3eb791df0ae727e8dbc0259ed1249998d4"}, @@ -2617,7 +2226,7 @@ description = "The next generation HTTP client." optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" +markers = "extra == \"vllm\" or extra == \"tinker\"" files = [ {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, @@ -2644,7 +2253,7 @@ description = "Consume Server-Sent Event (SSE) messages with HTTPX." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc"}, {file = "httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d"}, @@ -2689,24 +2298,6 @@ testing = ["InquirerPy (==0.3.4)", "Jinja2", "Pillow", "aiohttp", "authlib (>=1. torch = ["safetensors[torch]", "torch"] typing = ["types-PyYAML", "types-requests", "types-simplejson", "types-toml", "types-tqdm", "types-urllib3", "typing-extensions (>=4.8.0)"] -[[package]] -name = "hydra-core" -version = "1.3.7" -description = "A framework for elegantly configuring complex applications" -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "hydra_core-1.3.7-py3-none-any.whl", hash = "sha256:83606c4a05c770f1e026e0489875f28989829d151799eabd8cfa8cbe82164515"}, - {file = "hydra_core-1.3.7.tar.gz", hash = "sha256:d4ff629c7e23b64a9db5fe5b027cf7c55b633b737f03be51ef41b77ed016f75e"}, -] - -[package.dependencies] -antlr4-python3-runtime = "==4.9.*" -omegaconf = ">=2.2,<2.4" -packaging = "*" - [[package]] name = "hyperframe" version = "6.1.0" @@ -2714,126 +2305,12 @@ description = "Pure-Python HTTP/2 framing" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "python_version >= \"3.11\" and extra == \"client\"" +markers = "extra == \"tinker\"" files = [ {file = "hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5"}, {file = "hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08"}, ] -[[package]] -name = "hypothesis" -version = "6.168.0" -description = "The property-based testing library for Python" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"test\"" -files = [ - {file = "hypothesis-6.168.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:47b89491ff02e3ae9b302c440457938e87b47a45b9a1d98ff5575b6910d779e2"}, - {file = "hypothesis-6.168.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:1f4cd0ff11bd470a1a846296ed5fe55e84214194850370994fd1370fe73d3099"}, - {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:732ae5d47482f99d8028cca096729625f05690a83f5e7ce31466e266155792f4"}, - {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2085ee74ac3ab6b70e2f7ffae9b4cb74c246da2f574b2de81a0818a8a30f659f"}, - {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1894782fae5d9a7bb44e6dcf848ccb09ccb5babab48d8b5c31a0a7fc025b82a1"}, - {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecf0ab13cef899efb816ffdd7963e0679f372520884ce06756c7642f3df94213"}, - {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:3f6dcf66270278d078bed01b401f47db4e26456cd909d8e23c6b9366a6c0b131"}, - {file = "hypothesis-6.168.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bfef4d46dbf1704a7b8fa3a78778651a2cb18870ca0a70da19c381646822b149"}, - {file = "hypothesis-6.168.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1d1aa5b3484e329295d88488a5ba06243909e65c2ab616513c2d36721de4ed1d"}, - {file = "hypothesis-6.168.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:3bc00fd8cda04b58e37a1163e8a65389b247b4f5ee547ae37d244a4960995517"}, - {file = "hypothesis-6.168.0-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:990026952d5b2eca290c88f639ac639233f47e13dae338c6dfb6e4774bcab349"}, - {file = "hypothesis-6.168.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:a74b0945acbbd552c7c2d0a99a3b5232962b8848c8eed1829451800a9bfcf00b"}, - {file = "hypothesis-6.168.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2a380b521b5a76a9e8917d64adcf7f861a45a4360a34b1579af14c5df8eb0377"}, - {file = "hypothesis-6.168.0-cp310-abi3-win32.whl", hash = "sha256:2264f15a1c80329e3ad48e39c44bd5c9429b7b04c9ee62cdd72f4b10aaac9f29"}, - {file = "hypothesis-6.168.0-cp310-abi3-win_amd64.whl", hash = "sha256:5b54769033b84477931d2072e7133a7555e0de5c53fd5ca3bbde960762d7d31b"}, - {file = "hypothesis-6.168.0-cp310-abi3-win_arm64.whl", hash = "sha256:112b0900059bf9d7d6528ed729770629ab146e0d133c4143b9bd4a01dc002bcc"}, - {file = "hypothesis-6.168.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:cb10aa59b0af45badca76911f5323f40d24fdbe00d01b7b67fef8648c99411b5"}, - {file = "hypothesis-6.168.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e21e30b76b6d3adb87c550576132a3204f4c257ec43353f6c09b9d59bb762abc"}, - {file = "hypothesis-6.168.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:527452b43e79e6dfbf9cb69145a940547a3cd177c556698a3fc939ed2354c4b3"}, - {file = "hypothesis-6.168.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:489d5c060f49f495b64215cae627c71730cffd5ef59dc4d7f431932e6e6d2e67"}, - {file = "hypothesis-6.168.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c3af200b322f710c76c2189866246cdcff2039165dd77edff1a7bf1157162fb0"}, - {file = "hypothesis-6.168.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:73084b76e4a79cd0f7883ce80fc60c9f374ce7dcad8f520b39db40470ce1852f"}, - {file = "hypothesis-6.168.0-cp310-cp310-win_amd64.whl", hash = "sha256:8067e6b4b48e5cfdc849a1a20c9d4972b3f532b3e3edb5e2b5dfd106045a5236"}, - {file = "hypothesis-6.168.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:4d7d29dd63ad9fdc4aa1d65fa272449e14aaf6c6bb8451091818c2945533a43a"}, - {file = "hypothesis-6.168.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9a72ed7afa1f7e30488b8a5754fca0ad9755518bdb77d6f0b003cadf7437a5f9"}, - {file = "hypothesis-6.168.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fcc5bad4300a751804ce41f0e10d77f85272668160708ce39ec579bca8984843"}, - {file = "hypothesis-6.168.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53469a1a7c4861b12c9a8622f762d7d1fd7bcf171884e1018ed5a8f063a5c063"}, - {file = "hypothesis-6.168.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a9650c4882fdbdd8e90bdae602a8bfa8c6f09dc5d06afec5b9b23982e8f60a04"}, - {file = "hypothesis-6.168.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:348d9b93fd4129f67f9bab94f3d70709a9372bbe0e0d22731325ce85d5eb409f"}, - {file = "hypothesis-6.168.0-cp311-cp311-win_amd64.whl", hash = "sha256:719b45b0512e3535a6a0077c2f7c6053b02ac0e72d60693f66f98790a33855b2"}, - {file = "hypothesis-6.168.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:6b750390dac4429da0cb70ab3fe758457f0cea3d9c843d48c59d0690d1189fda"}, - {file = "hypothesis-6.168.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8e4b2d434e0dd134f3d31ac1efc1825bf99730dfe70fec005ff66d7211836d79"}, - {file = "hypothesis-6.168.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d4d36ed2fd62de11382f1d608169c1ffa9a49d3b9351146d8ff87cb81a66f7"}, - {file = "hypothesis-6.168.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5920d267f7d8cfd376672f2bde5905cdf284d47519582e41ce7c142d48ee46c4"}, - {file = "hypothesis-6.168.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fb8cdf45361e259df86e19f8cd042ce2d6c7e6ad88fa631b78a4e3a83c2e572d"}, - {file = "hypothesis-6.168.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3b3ce1cce70b25a37ed1a38a53ce7204785726c675c0f41a0f83c338a7e47b3d"}, - {file = "hypothesis-6.168.0-cp312-cp312-win_amd64.whl", hash = "sha256:f62bdabf278db9ff61df5f3203d608949f0d893d0e30cdac3f2330e67e41ae68"}, - {file = "hypothesis-6.168.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:7d55562bf8d41cfa18559c33f30cadf44ceac8e517509d7a022a9feace621f28"}, - {file = "hypothesis-6.168.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:92cff497b92e2285ff6a94193fdee04aba483a4115d501c1f9a570bd103fcd20"}, - {file = "hypothesis-6.168.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ff259260015f9be3756dcd4bc11c08e007314dec6b43d9a89084c4f34f94475"}, - {file = "hypothesis-6.168.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35f1262831b5acc74ded15f629965daffcd657f6016ee04fc9605f6eb2b334c0"}, - {file = "hypothesis-6.168.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:046fe4bcfce2a2fa186ba9d96bbb62c25c2f6c2e4071f0783ed6b5cc481d0669"}, - {file = "hypothesis-6.168.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:24b52a2b1c8db6e1e516f9295c8e4ef7ef63303ff24fbbc5b35f4ff71dcd732c"}, - {file = "hypothesis-6.168.0-cp313-cp313-win_amd64.whl", hash = "sha256:ec0886fe0be9091669937989f9a662beca42ae14a4a6dab25491c2c63365f88d"}, - {file = "hypothesis-6.168.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:e2df8afacf9261070795db36db4a394e3ccdbb663fd2d38c7a9fba0c836dcecc"}, - {file = "hypothesis-6.168.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:9ba679f183c67adcb6f4ad93694beafb6da99fe691757f4e57b04ae77e581ba8"}, - {file = "hypothesis-6.168.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d9a8574f80fc859313aee56167d202e8625c0eedd200971130f0839f06d1c93"}, - {file = "hypothesis-6.168.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:deb02de608268928d779aa889b0a9d67794b1cc0c54a322cf19e386be8a46ca7"}, - {file = "hypothesis-6.168.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:076a2096c34448931c3cfeb2eb7a6b843a56ffdce5e4e3a025bfdf8f935666d9"}, - {file = "hypothesis-6.168.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5f099b1c8fc49ec2d9d7944e661addb97d7c38e818fb8d1f78073c43895a87f6"}, - {file = "hypothesis-6.168.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:93413d1b0af50a7b165d66278c529174bf2fd1773c78027735dc0b50d1d3fd27"}, - {file = "hypothesis-6.168.0-cp314-cp314-win_amd64.whl", hash = "sha256:db2751c27bffc8491a96d72969649089d5400115e4b7c49bf7167ebbdcc84193"}, - {file = "hypothesis-6.168.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:cd0c1dcf308e919c8ae708054d0ad61921ae87634a9aea574a9851da584cebc1"}, - {file = "hypothesis-6.168.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d0bdb77f976740b8cd5ec697327ea343d02d052b9916d213b5d4c65d823415cd"}, - {file = "hypothesis-6.168.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3f7486bed33225d02f6aa78a4c4ba2b6f84992a82571cdda1bf08dce41d13507"}, - {file = "hypothesis-6.168.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ba3838c4a92e0b9730d1ed7e67e4950c152ad79d0a0c7594065262db84c55c4"}, - {file = "hypothesis-6.168.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:891b2d281ede45130e7fa0a22fd65336cc77ef2f780ec3792e8de6fc274a02c8"}, - {file = "hypothesis-6.168.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e86820053afad84677f301c0b892a226be1df49790800a65668ae7cc8a1ac571"}, - {file = "hypothesis-6.168.0-cp314-cp314t-win_amd64.whl", hash = "sha256:a4956f41ab1ec6e6ef9262a35970e9f3e2caaaa1cdafe0d413156c6934dd99d8"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:754016594fe78cef91790e0922f60d183c52f531255fbfa30dac495b813e2128"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:6f0dd437ec01140676192422b61f2f833b3ce6a3213da9b7e196ad6b3777e795"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f77af7721ff35a58fa8797decd14c932c350a2548686c6e9b844db710a3a2441"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a0d28418c104d7268fdebcc09bc49f7b6569b5eb942430c6859f53ec8d4edf63"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:812a84c4cc7f7ae4fcb39a5647cc2698e6c18254f8423126425578f1dcdac782"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6de30e559eb151de14a5f74bceb4d97792a9315ada2a1816b5da825cd7d28edc"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:9018b20acdb061b2ef4b2fa7f558ca5db97ffea316e0a528bc003a24b2ac996e"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:bc935a5d5f86fd8f5af951b8fbe00307f6f7c596f82a9a27c17d974f6ab0a26c"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:45fcfa05f746e253350f55f216bcef59754f5f2b85745f1fc2bb8ba81dd517a9"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:f89d8e998d3c936ffbbd1c3686c96f0378f6558aecc5967a3035a857f2bab0ad"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:d0620fa320fa66649e6bfd71e94f3f86115fffebb7e3c6dcece19d1aaff8e07f"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:4085b61e25d3dcc6c9151d4115269870aee8cdb921611ee5c989b2786449be09"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:b5449a64eb37d9a4aa6ac9cd2ab0fd1a24145adf421ef1536884f73f39824887"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-win32.whl", hash = "sha256:91e3de666a6c4f7543000d1710e25055d63ef3032c98bd2ab338b3087bdaa780"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:9a2079cd09919956dd388f1a1f8ea5a79f2b2437650fbeda31d8661217ffefef"}, - {file = "hypothesis-6.168.0-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:085c9aa246487c56a40ca89003d285cbffdbb5be4097ba6d0139f9c21003c04a"}, - {file = "hypothesis-6.168.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:16864797de4b024e4c6cebd44598af932f870aad811341bc5bc24c738801ff76"}, - {file = "hypothesis-6.168.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:283eda952bcb1987ccba1c8b634db0e8a960e1e92e2daa7003bc2392f19cea01"}, - {file = "hypothesis-6.168.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5427a3c951080c18170486f775df6a82153882b819eca6b8e7ed77693634e5ab"}, - {file = "hypothesis-6.168.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a838218ff1eab8d7b4bf66b96037fce0a802f61f2fa5fd4b784696cac365ce7"}, - {file = "hypothesis-6.168.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:34e3c8b66047ba92f8b8df5e427074058d92db58038f007da4bf9d14e934ad3c"}, - {file = "hypothesis-6.168.0.tar.gz", hash = "sha256:72af51087b7b5ab21c49f0d502f803c20897678652835596bd2a8b169a39135e"}, -] - -[package.dependencies] -exceptiongroup = {version = ">=1.0.0", markers = "python_full_version < \"3.11.0\""} -sortedcontainers = ">=2.1.0,<3.0.0" - -[package.extras] -all = ["black (>=20.8b0)", "click (>=7.0)", "crosshair-tool (>=0.0.110)", "django (>=5.2)", "dpcontracts (>=0.4)", "hypothesis-crosshair (>=0.0.30)", "lark (>=0.10.1)", "libcst (>=0.3.16)", "numpy (>=1.21.6)", "pandas (>=1.1)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "tzdata (>=2026.3) ; sys_platform == \"emscripten\" or sys_platform == \"win32\"", "watchdog (>=4.0.0)"] -cli = ["black (>=20.8b0)", "click (>=7.0)", "rich (>=9.0.0)"] -codemods = ["libcst (>=0.3.16)"] -crosshair = ["crosshair-tool (>=0.0.110)", "hypothesis-crosshair (>=0.0.30)"] -dateutil = ["python-dateutil (>=1.4)"] -django = ["django (>=5.2)"] -dpcontracts = ["dpcontracts (>=0.4)"] -ghostwriter = ["black (>=20.8b0)"] -lark = ["lark (>=0.10.1)"] -numpy = ["numpy (>=1.21.6)"] -pandas = ["pandas (>=1.1)"] -pytest = ["pytest (>=4.6)"] -pytz = ["pytz (>=2014.1)"] -redis = ["redis (>=3.0.0)"] -watchdog = ["watchdog (>=4.0.0)"] -zoneinfo = ["tzdata (>=2026.3) ; sys_platform == \"emscripten\" or sys_platform == \"win32\""] - [[package]] name = "idna" version = "3.11" @@ -2856,7 +2333,7 @@ description = "Iterative JSON parser with standard Python iterator interfaces" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "ijson-3.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ea8dcac10d86adaeead454bc25c97b68d0bda573d5fd6f86f5e21cf8f7906f88"}, {file = "ijson-3.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:92b0495bbb2150bbf14fc5d98fb6d76bcd1c526605a172709e602e6fedc96495"}, @@ -2975,7 +2452,7 @@ description = "Read metadata from Python packages" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "(python_version >= \"3.11\" or extra == \"eval\" or extra == \"megatron\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\") and (python_version == \"3.11\" or extra == \"eval\" or extra == \"megatron\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\" or extra == \"client\") and (extra == \"client\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\" or extra == \"eval\" or extra == \"megatron\")" +markers = "extra == \"megatron\" or extra == \"ray\"" files = [ {file = "importlib_metadata-8.7.1-py3-none-any.whl", hash = "sha256:5a1f80bf1daa489495071efbb095d75a634cf28a8bc299581244063b53176151"}, {file = "importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb"}, @@ -2993,19 +2470,6 @@ perf = ["ipython"] test = ["flufl.flake8", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] type = ["mypy (<1.19) ; platform_python_implementation == \"PyPy\"", "pytest-mypy (>=1.0.1)"] -[[package]] -name = "iniconfig" -version = "2.3.0" -description = "brain-dead simple config-ini parsing" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"test\"" -files = [ - {file = "iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12"}, - {file = "iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730"}, -] - [[package]] name = "interegular" version = "0.3.3" @@ -3013,24 +2477,12 @@ description = "a regex intersection checker" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "interegular-0.3.3-py37-none-any.whl", hash = "sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c"}, {file = "interegular-0.3.3.tar.gz", hash = "sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600"}, ] -[[package]] -name = "jieba" -version = "0.42.1" -description = "Chinese Words Segmentation Utilities" -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "jieba-0.42.1.tar.gz", hash = "sha256:055ca12f62674fafed09427f176506079bc135638a14e23e25be909131928db2"}, -] - [[package]] name = "jinja2" version = "3.1.6" @@ -3056,7 +2508,7 @@ description = "Fast iterable JSON parser." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" +markers = "extra == \"vllm\"" files = [ {file = "jiter-0.13.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:2ffc63785fd6c7977defe49b9824ae6ce2b2e2b77ce539bdaf006c26da06342e"}, {file = "jiter-0.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4a638816427006c1e3f0013eb66d391d7a3acda99a7b0cf091eff4497ccea33a"}, @@ -3169,48 +2621,12 @@ description = "JSON Matching Expressions" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64"}, {file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"}, ] -[[package]] -name = "joblib" -version = "1.6.0" -description = "Lightweight pipelining with Python functions" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "joblib-1.6.0-py3-none-any.whl", hash = "sha256:3dbbf9f6e4b592a2357b854608e980fe6390d131d7a82f011a377ef2ebef7aba"}, - {file = "joblib-1.6.0.tar.gz", hash = "sha256:2ccc96785b12046c08fd6d55839c12857831b54a3c1673ffadd2f04bfc4eda03"}, -] - -[package.dependencies] -cloudpickle = ">=3.0" - -[package.extras] -docs = ["distributed", "lz4", "matplotlib", "numpy", "numpydoc", "pandas", "psutil", "pydata-sphinx-theme", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx-gallery", "tqdm"] -test = ["distributed", "lz4", "memory_profiler", "numpy", "pytest", "pytest-asyncio", "pytest-cov", "pytest-run-parallel", "pytest-timeout", "threadpoolctl"] - -[[package]] -name = "jsonlines" -version = "4.0.0" -description = "Library with helpers for the jsonlines file format" -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "jsonlines-4.0.0-py3-none-any.whl", hash = "sha256:185b334ff2ca5a91362993f42e83588a360cf95ce4b71a73548502bda52a7c55"}, - {file = "jsonlines-4.0.0.tar.gz", hash = "sha256:0c6d2c09117550c089995247f605ae4cf77dd1533041d366351f6f298822ea74"}, -] - -[package.dependencies] -attrs = ">=19.2.0" - [[package]] name = "jsonschema" version = "4.26.0" @@ -3218,7 +2634,7 @@ description = "An implementation of JSON Schema validation for Python" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce"}, {file = "jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326"}, @@ -3226,7 +2642,7 @@ files = [ [package.dependencies] attrs = ">=22.2.0" -jsonschema-specifications = ">=2023.03.6" +jsonschema-specifications = ">=2023.3.6" referencing = ">=0.28.4" rpds-py = ">=0.25.0" @@ -3241,7 +2657,7 @@ description = "The JSON Schema meta-schemas and vocabularies, exposed as a Regis optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe"}, {file = "jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d"}, @@ -3251,124 +2667,47 @@ files = [ referencing = ">=0.31.0" [[package]] -name = "lark" -version = "1.2.2" -description = "a modern parsing library" -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" -files = [ - {file = "lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c"}, - {file = "lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80"}, -] - -[package.extras] -atomic-cache = ["atomicwrites"] -interegular = ["interegular (>=0.3.1,<0.4.0)"] -nearley = ["js2py"] -regex = ["regex"] - -[[package]] -name = "latex2sympy2-extended" -version = "1.10.2" -description = "Convert LaTeX math to SymPy expressions" +name = "kernels" +version = "0.12.1" +description = "Download compute kernels" optional = true -python-versions = ">=3.10" +python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"eval\" or extra == \"async-rl\"" +markers = "extra == \"kernels\"" files = [ - {file = "latex2sympy2_extended-1.10.2-py3-none-any.whl", hash = "sha256:f910442c5b02a466c1046f47d05cc5285181068b882399281f30102715337fb7"}, - {file = "latex2sympy2_extended-1.10.2.tar.gz", hash = "sha256:41a517ffcc5a140e910a7d1646ce6ff440817e5f9d48fc8279d88bd0925bc389"}, + {file = "kernels-0.12.1-py3-none-any.whl", hash = "sha256:4569fe00bc6a2b0f2cccff81841d7917af143c786ade152c3b2a78f61097c757"}, + {file = "kernels-0.12.1.tar.gz", hash = "sha256:41e31ef167add0062cdc302a943ce287194c13b5af12b82a5fa76e7353ca0042"}, ] [package.dependencies] -antlr4-python3-runtime = [ - {version = ">=4.9.3,<=4.13.2"}, - {version = "4.9.3", optional = true, markers = "extra == \"antlr4-9-3\""}, -] -sympy = "*" +huggingface_hub = ">=0.26.0,<2.0" +packaging = ">=20.0" +pyyaml = ">=6" [package.extras] -antlr4-11-0 = ["antlr4-python3-runtime (==4.11.0)"] -antlr4-13-2 = ["antlr4-python3-runtime (==4.13.2)"] -antlr4-9-3 = ["antlr4-python3-runtime (==4.9.3)"] -dev = ["pytest"] +abi-check = ["kernel-abi-check (>=0.6.2,<0.7.0)"] +benchmark = ["numpy (>=2.0.2)", "requests (>=2.32.5)", "torch"] +docs = ["hf-doc-builder"] +torch = ["torch"] [[package]] -name = "linkify-it-py" -version = "2.2.0" -description = "Links recognition library with FULL unicode support." +name = "lark" +version = "1.2.2" +description = "a modern parsing library" optional = true -python-versions = ">=3.10" +python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"client\"" +markers = "extra == \"vllm\"" files = [ - {file = "linkify_it_py-2.2.0-py3-none-any.whl", hash = "sha256:3adc40eb5af300b2605fcfdb968c24e1d780a90f1f2221af7c15e5111e94d443"}, - {file = "linkify_it_py-2.2.0.tar.gz", hash = "sha256:907acd2d17ac1fbb9ddb62c8957ccbd6158cac602231a15c3b0cd1e215f03cee"}, + {file = "lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c"}, + {file = "lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80"}, ] [package.extras] -benchmark = ["pytest", "pytest-benchmark"] -dev = ["black", "flake8", "isort", "pre-commit", "pyproject-flake8"] -doc = ["myst-parser", "sphinx", "sphinx_book_theme"] -test = ["coverage", "pytest", "pytest-cov", "pytest-timeout"] - -[[package]] -name = "litellm" -version = "1.95.0" -description = "Library to easily interface with LLM API providers" -optional = true -python-versions = "<3.15,>=3.10" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "litellm-1.95.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:0106b3564b60d00cb5b2810824ebf5071f59c9f9262318884d9c6f040aa2c435"}, - {file = "litellm-1.95.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:053cea1a584bf92d5d44b422f99fa03715bf7c346c8fe080c29bf0c6bd71eddc"}, - {file = "litellm-1.95.0-cp310-cp310-win_amd64.whl", hash = "sha256:667cc7cc58e05a9f9c4bf4588cd4ff5c785fd265060acdfb9147332b75b73ce9"}, - {file = "litellm-1.95.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4bafa3494d503a3c6c1f2eeb785a2af364d85463c71a92cabaaa761c9227d62a"}, - {file = "litellm-1.95.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:4c2a06d2263a07a29228cd3af2b594cf86cd3a4182a7324c517cba88a9415969"}, - {file = "litellm-1.95.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac37bb6d2be191bafc0ce590ed06d21dd7e5a37599ffc1af1071899f6c74b73"}, - {file = "litellm-1.95.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:cb667f84f08520f32b076e03c7a3fa51bf3f7e8b641dade34ab046bf00314d6b"}, - {file = "litellm-1.95.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:1bdf7153557cc0851fa9477b137fde476c56d5de92a5778ecfc6c3a75439a4e1"}, - {file = "litellm-1.95.0-cp312-cp312-win_amd64.whl", hash = "sha256:62cc5d834e8223dbd16c9ad0b46c73354b6d67cc7fa0eba2764ce65b3b8c474f"}, - {file = "litellm-1.95.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:9d80a9adc506bfce48145621d6649e3fd428407811eb00211bcce33344054701"}, - {file = "litellm-1.95.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cf014ff515825ad49937b4cdf95616270789311db7841d16702e0a5b1ac5b067"}, - {file = "litellm-1.95.0-cp313-cp313-win_amd64.whl", hash = "sha256:c73df441153e585832d4e90e3717d17ae888b269daa71d723336369e81ef884b"}, - {file = "litellm-1.95.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:c8e712f95764a9a730f3aec9dff8be916973423411ca12a79a35c8bea3f7d5a4"}, - {file = "litellm-1.95.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:071a63c0e1d949bf7ed5e5c73843cb042a6e00324d3f214b13f8c6b90da6822d"}, - {file = "litellm-1.95.0-cp314-cp314-win_amd64.whl", hash = "sha256:c3684dcf16aefe98bd6f11586d60a9a92bb1bf7e85468a6a4ac28a65532fab3e"}, - {file = "litellm-1.95.0.tar.gz", hash = "sha256:0ef126d52c7a559f8353e50d60fd0d5e7e6c8767ad54df25ddaf79b9edca1afc"}, -] - -[package.dependencies] -aiohttp = ">=3.10,<4.0" -click = ">=8.0.0,<9.0" -fastuuid = ">=0.14.0,<1.0" -httpx = ">=0.28.0,<1.0" -importlib-metadata = ">=8.0.0,<9.0" -jinja2 = ">=3.1.6,<4.0" -jsonschema = ">=4.0.0,<5.0" -openai = ">=2.20.0,<3.0.0" -pydantic = ">=2.10.0,<3.0.0" -python-dotenv = ">=1.0.0,<2.0" -tiktoken = ">=0.8.0,<1.0" -tokenizers = ">=0.21.0,<1.0" - -[package.extras] -bedrock-realtime = ["aws-sdk-bedrock-runtime (>=0.7.0,<0.8.0) ; python_full_version >= \"3.12.0\""] -caching = ["diskcache (>=5.6.3,<6.0)"] -cli = ["inquirerpy (>=0.3.4,<1.0)", "pyyaml (>=6.0.3,<7.0)", "requests (>=2.32.0,<3.0)", "rich (>=13.9.4,<14.0)"] -extra-proxy = ["a2a-sdk (>=1.1.0,<2.0)", "azure-identity (>=1.25.2,<2.0)", "azure-keyvault-secrets (>=4.10.0,<5.0)", "google-cloud-iam (>=2.19.1,<3.0)", "google-cloud-kms (>=2.24.2,<3.0)", "prisma (>=0.11.0,<1.0)", "redisvl (>=0.4.1,<1.0)", "resend (>=2.23.0,<3.0)"] -google = ["google-cloud-aiplatform (>=1.133.0,<2.0)"] -grpc = ["grpcio (==1.78.0)"] -mlflow = ["mlflow (>=3.11.1,<4.0)"] -proxy = ["apscheduler (>=3.11.2,<4.0)", "azure-identity (>=1.25.2,<2.0)", "azure-storage-blob (>=12.28.0,<13.0)", "backoff (>=2.2.1,<3.0)", "boto3 (>=1.43.1,<2.0)", "cryptography (>=48.0.1,<49.0)", "expression (>=5.6.0,<6.0)", "fastapi (>=0.136.3,<1.0)", "fastapi-sso (>=0.19.0,<1.0)", "granian (>=2.7.4,<3.0)", "gunicorn (>=23.0.0,<24.0)", "inquirerpy (>=0.3.4,<1.0)", "litellm-enterprise (==0.1.52)", "litellm-proxy-extras (==0.4.81)", "mcp (>=1.28.1,<2.0)", "orjson (>=3.11.6,<4.0)", "polars (>=1.38.1,<2.0)", "pydantic-settings (>=2.14.1,<3.0)", "pyjwt (>=2.13.0,<3.0)", "pynacl (>=1.6.2,<2.0)", "pyroscope-io (>=0.8.16,<1.0) ; sys_platform != \"win32\"", "python-multipart (>=0.0.27,<1.0)", "pyyaml (>=6.0.3,<7.0)", "restrictedpython (>=8.1,<9.0)", "rich (>=13.9.4,<14.0)", "rq (>=2.7.0,<3.0)", "soundfile (>=0.12.1,<1.0)", "starlette (>=1.0.1,<2.0)", "uvicorn (>=0.33.0,<1.0)", "uvloop (>=0.21.0,<1.0) ; sys_platform != \"win32\"", "websockets (>=15.0.1,<16.0)"] -proxy-runtime = ["anthropic[vertex] (>=0.84.0,<1.0)", "azure-ai-contentsafety (>=1.0.0,<2.0)", "azure-storage-file-datalake (>=12.20.0,<13.0)", "ddtrace (>=4.8.2,<5.0)", "detect-secrets (>=1.5.0,<2.0)", "google-cloud-aiplatform (>=1.133.0,<2.0)", "google-genai (>=1.37.0,<2.0)", "grpcio (==1.78.0)", "langfuse (>=2.59.7,<3.0)", "llm-sandbox (>=0.3.39,<1.0)", "mangum (>=0.17.0,<1.0)", "opentelemetry-api (==1.28.0)", "opentelemetry-exporter-otlp (==1.28.0)", "opentelemetry-instrumentation-fastapi (==0.49b0)", "opentelemetry-sdk (==1.28.0)", "prometheus-client (>=0.20.0,<1.0)", "pypdf (>=6.12.0,<7.0)", "sentry-sdk (>=2.21.0,<3.0)"] -saml = ["python3-saml (>=1.16.0,<2.0)"] -semantic-router = ["aurelio-sdk (>=0.0.19,<1.0) ; python_full_version < \"3.14.0\"", "semantic-router (>=0.1.15,<1.0) ; python_full_version < \"3.14.0\""] -stt-nvidia-riva = ["audioread (>=3.0.1)", "numpy (>=1.26.0)", "nvidia-riva-client (>=2.15.0)", "soundfile (>=0.12.1)"] -utils = ["numpydoc (>=1.8.0,<2.0)"] +atomic-cache = ["atomicwrites"] +interegular = ["interegular (>=0.3.1,<0.4.0)"] +nearley = ["js2py"] +regex = ["regex"] [[package]] name = "llguidance" @@ -3377,7 +2716,7 @@ description = "Bindings for the Low-level Guidance (llguidance) Rust library for optional = true python-versions = ">=3.9" groups = ["main"] -markers = "(extra == \"rl\" or extra == \"async-rl\") and (platform_machine == \"x86_64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\")" +markers = "extra == \"vllm\" and (platform_machine == \"x86_64\" or platform_machine == \"arm64\" or platform_machine == \"aarch64\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\")" files = [ {file = "llguidance-1.3.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f7685222660a762e481ac633d49cc559c64980fe2ee59c8f932a5bb5cbc0c2c2"}, {file = "llguidance-1.3.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:098030ff0687261a3f1bd54cf21fe951fc861d56d37a0671250dd36677eaf224"}, @@ -3396,7 +2735,7 @@ description = "lightweight wrapper around basic LLVM functionality" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "llvmlite-0.44.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:9fbadbfba8422123bab5535b293da1cf72f9f478a65645ecd73e781f962ca614"}, {file = "llvmlite-0.44.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cccf8eb28f24840f2689fb1a45f9c0f7e582dd24e088dcf96e424834af11f791"}, @@ -3428,7 +2767,7 @@ description = "Enforce the output format (JSON Schema, Regex etc) of a language optional = true python-versions = "<4.0,>=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "lm_format_enforcer-0.11.3-py3-none-any.whl", hash = "sha256:cf586350875def1ae7a8fba84fcbbfc8371424b6c9d05c1fcba70aa233fbf06f"}, {file = "lm_format_enforcer-0.11.3.tar.gz", hash = "sha256:e68081c108719cce284a9bcc889709b26ffb085a1945b5eba3a12cfa96d528da"}, @@ -3447,7 +2786,7 @@ description = "Python logging made (stupidly) simple" optional = true python-versions = "<4.0,>=3.5" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c"}, {file = "loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6"}, @@ -3458,200 +2797,7 @@ colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] -dev = ["Sphinx (==8.1.3) ; python_version >= \"3.11\"", "build (==1.2.2) ; python_version >= \"3.11\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.5.0) ; python_version >= \"3.8\"", "mypy (==v0.910) ; python_version < \"3.6\"", "mypy (==v0.971) ; python_version == \"3.6\"", "mypy (==v1.13.0) ; python_version >= \"3.8\"", "mypy (==v1.4.1) ; python_version == \"3.7\"", "myst-parser (==4.0.0) ; python_version >= \"3.11\"", "pre-commit (==4.0.1) ; python_version >= \"3.9\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==8.3.2) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==5.0.0) ; python_version == \"3.8\"", "pytest-cov (==6.0.0) ; python_version >= \"3.9\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.1.0) ; python_version >= \"3.8\"", "sphinx-rtd-theme (==3.0.2) ; python_version >= \"3.11\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.23.2) ; python_version >= \"3.8\"", "twine (==6.0.1) ; python_version >= \"3.11\""] - -[[package]] -name = "lxml" -version = "6.1.3" -description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "lxml-6.1.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:40bcbd9f94166ffe925811e730607385cec959f42fb1bb7dad83748680465221"}, - {file = "lxml-6.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:05f5bce9af14fd1506997594bd81cee6d9c6b58ea80a39c058327aa6371ed9e9"}, - {file = "lxml-6.1.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ff88a92cafde90888511242d1c54afcc1a8adbb6dc0a88fa7f87e29e92400d4a"}, - {file = "lxml-6.1.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c00e26288784460885fe76e4d4b293573e0f791f52e6d60e27b42edf005922eb"}, - {file = "lxml-6.1.3-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:773062aec2f2e56b2b22d37054123f0de8a22a4688a0c3376c3fe42685f975cf"}, - {file = "lxml-6.1.3-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f6449672f9c93316deb5e2839e18931f468670e44d5bd9b1301a5a9655d45c07"}, - {file = "lxml-6.1.3-cp310-cp310-manylinux_2_28_i686.whl", hash = "sha256:ec295280f4b37769256da025acf5890370355ac589c27e89caae0b5e9eedc702"}, - {file = "lxml-6.1.3-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:5929d9df5e7e3379183be0e21f7d559618a5b61cb63280df6164019242e337ed"}, - {file = "lxml-6.1.3-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6e1eb8a4cbffd5553680ad96be6680e364710656eced73d1dc90ec489df599a3"}, - {file = "lxml-6.1.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:16148acd77ed1d8836a56db883af2f5eed720f9723088110b16a0d08582130a6"}, - {file = "lxml-6.1.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:23c366231259cd75ad06495174701afb3fcb36a92917fa47de2d1f1bd9d95739"}, - {file = "lxml-6.1.3-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:da85db328e507da922d586c3c7416ec360ec22e9cd9e0700691afacde0c81f53"}, - {file = "lxml-6.1.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0f17d83c48ee9dfd96abae3ac3e2108c76d2fc86ce96355e37b8da9f7f4ecc08"}, - {file = "lxml-6.1.3-cp310-cp310-win32.whl", hash = "sha256:7dd624c1eaa629ad44b59a1a0145fdf2d67895592dce94c9358b938b3d075e65"}, - {file = "lxml-6.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:18a4db52b5a7b53a3540b0b0f4123319334621ee8083d496de314d0bf06ff59a"}, - {file = "lxml-6.1.3-cp310-cp310-win_arm64.whl", hash = "sha256:0feebef8d0521188d0157f758356072e840173aa61ca45b8b3f87959ac283dd5"}, - {file = "lxml-6.1.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c66f858b82497173f73366795fc6ee8171620e75a338506d6b2e7bc16f5fca11"}, - {file = "lxml-6.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:032a0a97eed428bd143c75a11118238546424ceb2fa311cca5f073aa44658dc4"}, - {file = "lxml-6.1.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4a579dfb9c835f8ab47f4b8ed33440cbc75b806b73297208e6ec2a33e903740b"}, - {file = "lxml-6.1.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:49fbc2682a9306135b7ec49e93f97f9c26689b9b7f96ed2742d8d6497e994d13"}, - {file = "lxml-6.1.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ea2c01cdb16dc12156e455007c406dfaaece0c89aa4ba0e3b47586779f951d41"}, - {file = "lxml-6.1.3-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:527195c188d7d0af748cd48d220ab8cdc5cb99be3d49ac4d9be7324d8abf9bc0"}, - {file = "lxml-6.1.3-cp311-cp311-manylinux_2_28_i686.whl", hash = "sha256:20384c2bbcbf87180c8c61eb60869699c1ec0cd09b62cfd13804022d860b0867"}, - {file = "lxml-6.1.3-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:424aa5657141d306ba9ad1baab4b2c0a0719040075ee6c66aee9bb2dea2b5054"}, - {file = "lxml-6.1.3-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4736e6c87e603146d8949d8501da621ad20c31015060d3fcf95ace2859f3e3e6"}, - {file = "lxml-6.1.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6374e9e382e5a98c9c5e66d41b357b470da1c54bce30f17f9dc4bcc58436cc1c"}, - {file = "lxml-6.1.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:22eec57e26c418cde02c051ce9914a365e52a7f135a565c6f0480242aeebab48"}, - {file = "lxml-6.1.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8753b8d51dbc86fd335ee31fcf7f3658e9f5c016d4edfb23f76ad295f4b8c9d0"}, - {file = "lxml-6.1.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:207dfc3d47cf0e575e643bbc140dacc8863b39abaa1e5307cd64c7f2365b8a12"}, - {file = "lxml-6.1.3-cp311-cp311-win32.whl", hash = "sha256:18293f8a8d8b6a8e71ef37706b659e3846a4261232158167b1ddf35f6994f633"}, - {file = "lxml-6.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:7ae4949f212a53b007dbc355884fda122545c5764a54256c9217e419a62a6559"}, - {file = "lxml-6.1.3-cp311-cp311-win_arm64.whl", hash = "sha256:2123e5aa075ac20d23c7af489255efd129cbfe190dbe88fd42598cc9df3199b6"}, - {file = "lxml-6.1.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c0710ac085a157b593c38fbcacd950f15c4afa8e2057527185875ab302752bc"}, - {file = "lxml-6.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:623c8799c17128753c65699f1c3aa32402657393a9ad6db09ed8b98ddf76611d"}, - {file = "lxml-6.1.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f683dc6300317700025e41d89a43e0276692ded16113a3c43eab704d605c58e5"}, - {file = "lxml-6.1.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:379f8a75cf6eb7eef0af074b55f49ab73b868388a98de14646abcdfa4564bb11"}, - {file = "lxml-6.1.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b37772102d44bb6628186accca3a121b1fa3a6b3d97518a8c29a5229ca4c0d0a"}, - {file = "lxml-6.1.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddcf547bea2aee967d6a77779376a45e77e610e8465147a1f3d7e20d539d6e32"}, - {file = "lxml-6.1.3-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:909f4e927bb051f7740d6367285fc60cdcfdaf0258c2dba4ff5ba7eadadc250c"}, - {file = "lxml-6.1.3-cp312-cp312-manylinux_2_28_i686.whl", hash = "sha256:a5c18810318303ce9afb3f95e2ddb54834f96fa699a8600433fd5a93dcf44c56"}, - {file = "lxml-6.1.3-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:3e42265103fb385d8642a78672edf376c6f7e1d3598a7a4f9cb1278f2f6b5f6f"}, - {file = "lxml-6.1.3-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:21402998e4b78e7cce237d2788841aaa21ac9a4d1574d04dc2d12ee41ae807b5"}, - {file = "lxml-6.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:38fc4e4e4e084e0bd491949482527d406788045c546d4f8789e93fc527b91385"}, - {file = "lxml-6.1.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5609efdb0d3c95499c00046bc53648b3482ec2175b5503d6e611b3f0555dc71d"}, - {file = "lxml-6.1.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:97ce49699d87ebf8aad631b55d65b33219a4f1bfefbbf5bff19dc9af160aeaf9"}, - {file = "lxml-6.1.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:48542c9acba9ff9450bd18d871d2c2c8787fdb283572b623d206f1b927cd7d9e"}, - {file = "lxml-6.1.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c55e71a9b1db1f107efb60da49c093689b74c5c31a708e5379e2fd9439d4fbb5"}, - {file = "lxml-6.1.3-cp312-cp312-win32.whl", hash = "sha256:b3ff39654f0ce6ebd4db154211136dbe7e8157bcc3bed2344c87f32c7c6ecb6c"}, - {file = "lxml-6.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:3e9a00d1c2c30936f7add097c41afc5da6556c580909104aafd382cac92a855c"}, - {file = "lxml-6.1.3-cp312-cp312-win_arm64.whl", hash = "sha256:1aeca87830c4fe649dcf93fe2b059525b71c72587f21be4ae4af7103082a79fa"}, - {file = "lxml-6.1.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:3a48093cdb058a93af842ede9703520e810b05dcd0fc6d7190a06376c3bfb6bd"}, - {file = "lxml-6.1.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:887c021d9a977cff89cb273047c1352997b772a8908a25c21836861f69b92be1"}, - {file = "lxml-6.1.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:611a51e61c92f62345a50b0035df6fc0d678f9299f33728826d831598862f59d"}, - {file = "lxml-6.1.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b477912f42c5c33405a10c759d22f80cf5af043ae02d95b9d8e5e5bc555739ed"}, - {file = "lxml-6.1.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cffe18571ccc51d742cd08cbb3f8b756de9311d18c7ea98f5d92f37b8fb60c2"}, - {file = "lxml-6.1.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:75cc6569e86be5785b6188ef1642670c6adbc984e81ec35e224842ecd9eefcc8"}, - {file = "lxml-6.1.3-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d85dfab42dd672f87a7f76e9de7172962aee69fa12044f0d6e1a23cbd53fb80e"}, - {file = "lxml-6.1.3-cp313-cp313-manylinux_2_28_i686.whl", hash = "sha256:42632b4024ab24a6b488f559ac851312509888b6b80ae2aa11cf29a646a0d245"}, - {file = "lxml-6.1.3-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:febd35ef45f603c2d74b74655efdbf45e14f55fc0aef4ac82b663ca829b283e0"}, - {file = "lxml-6.1.3-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a43b3bdf11e477dc7770609d3477316f974354dfc8425d596f64f471cc8daf6e"}, - {file = "lxml-6.1.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d582042c69857c364e8153de6e18e0da9b7b515a6a8113caf69a6ec8e0520f2"}, - {file = "lxml-6.1.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:8e49a646acfab83c68974f4aa1d0a2acca9e88d7d627ae0fc13201b14b76d310"}, - {file = "lxml-6.1.3-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0dee106e9aa97fb00541b1ed7827070564d0549c3d3fba8920e6b20fd980f748"}, - {file = "lxml-6.1.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:dd5e90f34cffcfed97f36cf066325773d2b6021c60c29942e53a18b028501b1d"}, - {file = "lxml-6.1.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d9b3e7d71bf6acff341233417abbdface29c647e3113892d9aaedc02eb4aa2bc"}, - {file = "lxml-6.1.3-cp313-cp313-win32.whl", hash = "sha256:160fcf381f76c3aeac28a756bec44f48942a8f7245a87aa28e3a523b4d90cd87"}, - {file = "lxml-6.1.3-cp313-cp313-win_amd64.whl", hash = "sha256:e477aca0bc0d19f3b4ae9e4f2a1cfd687c31bf772d78734910658186b40b2477"}, - {file = "lxml-6.1.3-cp313-cp313-win_arm64.whl", hash = "sha256:b1cc980905221a5d8b3c476330730b3adb40ff80add71ffbdb6215ba055656f1"}, - {file = "lxml-6.1.3-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:2bec13085dc8ef48a3fe62f7dfcacfeda2c785cdf19cc8eeda2bb9ed081da165"}, - {file = "lxml-6.1.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:4f4db7c7e954d289d71878938348b3d91b904a3e8210a11939359fb758a58e7d"}, - {file = "lxml-6.1.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2cae5d5c90a62d9139c512a0cb1aad1d182b022b5740daea2617eb5bf7fc658e"}, - {file = "lxml-6.1.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c6c0c13128a32eb04a51357e56a094e13aa8e6d3d1884de2e9ae923f6915e1a8"}, - {file = "lxml-6.1.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2221e88679d1351e9a40aaee54bc65679b9795bbd0160bc3d5e36b163344eb75"}, - {file = "lxml-6.1.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cfb398886a7eb4c719161c3efcff2a1248febc53a4d8e5072d2d8a87fed84ac9"}, - {file = "lxml-6.1.3-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7eb78ba28b187e1e9203a55c60fcf70df2d22cb205fe6d51b9383d6097419f0"}, - {file = "lxml-6.1.3-cp314-cp314-manylinux_2_28_i686.whl", hash = "sha256:ea6b1e9105b4b24a34c722432d9fb578f9ed83af21fa1abda639011e0f22bbb6"}, - {file = "lxml-6.1.3-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:e8b17e23df3e827a69d25af70990ca2420e92668aaffaeeb3cd2351d7916a023"}, - {file = "lxml-6.1.3-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1b7c37339d7e75cab9a123a04248e243cefefb302ad6db566ea0c77cbcde421e"}, - {file = "lxml-6.1.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:83e3a51e7933db700a0da0db31849db3a24022d9970da9bb73001e1d0326fd92"}, - {file = "lxml-6.1.3-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9bde9ae026a55b9a192078dfa6e27dd0ca4a050171ab6272e92f97b757dfdf48"}, - {file = "lxml-6.1.3-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1a635e837b50a1819bebfedaac5916498ea024120969da8790500148fb0a894d"}, - {file = "lxml-6.1.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d0c5c362bc94f1929dc7e96e715bbe7bd17037f802e6d8f0d1545df9133c0559"}, - {file = "lxml-6.1.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c59e4265608da6a041f54646ecc0c9ecdbb19aaf14c4c684bb6c2114998cc415"}, - {file = "lxml-6.1.3-cp314-cp314-win32.whl", hash = "sha256:2e62c569ec7531b679b184cbfe335c501c1d13c4b363560013019962eb630e6d"}, - {file = "lxml-6.1.3-cp314-cp314-win_amd64.whl", hash = "sha256:66299564c046bc7e0cc5de5106601eae907e9fa5904cd68a323380a8502f7861"}, - {file = "lxml-6.1.3-cp314-cp314-win_arm64.whl", hash = "sha256:ebd054ad1737a68fb7c5c073d405cef2b88bb824e294de3b4a4e995b47f0e376"}, - {file = "lxml-6.1.3-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:5a143e6207579de8baeded4eaac9134413200359f1969d636f0bfb98ee8c3c8f"}, - {file = "lxml-6.1.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:a1cec0f99b9b914d39176347a93b7610dc09324491aee1cbc57cd291a41a1d55"}, - {file = "lxml-6.1.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f6b9d2aad499c769ee8287609ab0e6de99d8bcea99c6e6c2e64945259fd52fb2"}, - {file = "lxml-6.1.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a23fefdb345b2d4d0ff2860571b5ff9a89a28b6a120f720e8fb0324d346626"}, - {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:545ccc14fb05485f48b4439ec35beb16d5b5280eb6c81c658bd4707a2a119414"}, - {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:93476b6514b373fc6ca67d26c442784f7807c86f00635bfe79f935c3eab2af17"}, - {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8db38ff3fb7aee7d6a82ae4da2eef1178656fe1216841fbd24870062a9d60473"}, - {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_28_i686.whl", hash = "sha256:25f4118c438f96bb466e83108506d03d5c31b1bd2387e83e5b070bda6ded9c37"}, - {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:1beb0f9909b26cee938df9ba56b15252a84429b1fc30ce6fca161390b9789a70"}, - {file = "lxml-6.1.3-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3a27ac6c780c8b8a1cd231b58407634cafc1c4cc28cd6c7141362df0f36351e7"}, - {file = "lxml-6.1.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a1932d7ce78a561367512c594fe66eac2b2ec9b9264cfd9b5f950622f4a116e2"}, - {file = "lxml-6.1.3-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7d0f5976aa2701996f759b30172925829867547bb073af0ae67d1307a0f0262c"}, - {file = "lxml-6.1.3-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c5e7ce578aa8a80910a72a8ca0bbea3baae10100827249001999726a788456d8"}, - {file = "lxml-6.1.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:d97c5227621af74b111882a290b10f371780a38eef9d9e730408fba2259b52fb"}, - {file = "lxml-6.1.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da707f14ea3c35ee463d50acd596d6488e4b2b4ae7cf77a5bf93f55c023d63e8"}, - {file = "lxml-6.1.3-cp314-cp314t-win32.whl", hash = "sha256:9efe56a68179f3adc4de41861c9358931db03837c48dd5e1c78077b84dd07f3a"}, - {file = "lxml-6.1.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c9389b3784b56c58d933b5e0aecdf28f901b073ff385358d8a7d40907f6e14b2"}, - {file = "lxml-6.1.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32a409be3190b088f960ac92bfedfbef2f86c49ff940765e1548177592d20026"}, - {file = "lxml-6.1.3-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:6ea2f13dce778ca072ccee598bca46a092ce192e8fd907b6c1f0e52c800529a0"}, - {file = "lxml-6.1.3-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:c581b1d68b3845fb86c6b2983e755b29bf001461c59fa411d2c26a911b6559a9"}, - {file = "lxml-6.1.3-cp315-cp315-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e01125896585139453cab8cb235893644d8815d7509520da95ae3ee8d1c1f79"}, - {file = "lxml-6.1.3-cp315-cp315-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:290f66b97ede0e552e1cb44a0fd8a74f9753ee635b50830a0b122fb72788d015"}, - {file = "lxml-6.1.3-cp315-cp315-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73fc05988ed20809450474ba760a87c8ad4e455fc09783c02195e56ec634b41a"}, - {file = "lxml-6.1.3-cp315-cp315-manylinux_2_31_armv7l.whl", hash = "sha256:dc3a44689eea43eab836e5c98a8ab015dc2419987d1ea6eafc7c590cdff86bed"}, - {file = "lxml-6.1.3-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:209c3ccbfe35a04ac6d24f0611f9d1cbf8025d49991b14acd935236234d6c156"}, - {file = "lxml-6.1.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:2f5b2a2b9811b853b39bfa41367c6d78747b8e3e80e07fc5a24aae295c1a4d7d"}, - {file = "lxml-6.1.3-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:6a406d0b3cb207b0fa460ed4dc93e866f44f105da0169361cb18ff998a44c7f0"}, - {file = "lxml-6.1.3-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:53258656846f5c48996b882fb4b135885e088a3ad3d96b4bc0530f95124d1f69"}, - {file = "lxml-6.1.3-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:aa633613ff907ea91b9b0489a1f0da1b8725d8c6ccec6b77e8a1c9c235044bb0"}, - {file = "lxml-6.1.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:90f709b9accab6b2e4d14f5c8718203877a0486bcb3afd74d8b539ecd1e961d4"}, - {file = "lxml-6.1.3-cp315-cp315-win32.whl", hash = "sha256:b4fc6b03b9d9d90557274f571ab30e7fbbfc527955536935d96f98b6817a86e4"}, - {file = "lxml-6.1.3-cp315-cp315-win_amd64.whl", hash = "sha256:33cadd956b667997e4de1635fce9541f2e8ede2038fcde8cf55aa14d571d1bad"}, - {file = "lxml-6.1.3-cp315-cp315-win_arm64.whl", hash = "sha256:8a330c0ee5fa318c7b5cbbaad882baeca3f570357e7eb25ab34bf31008150758"}, - {file = "lxml-6.1.3-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:0bf5a3e397df2ec4258eb5eea4c1ac6cf013ca1abd04a176903bff20a70021fe"}, - {file = "lxml-6.1.3-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:13d22c0d57355366b393936acf6b98a5e0edeadddd3fccbc6a846c50a76b8741"}, - {file = "lxml-6.1.3-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cad7617727a96d189bd6f979d0fadf765198c7934e85f4edaba9bf3ad919a300"}, - {file = "lxml-6.1.3-cp315-cp315t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cae82b5ca24b0c2beedb269f6e2a96f466acd926879ab00ae19f1a65cbf9ffb0"}, - {file = "lxml-6.1.3-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:69cafd61aea04ebb3502c93c2aaa568b12931ca0802231e0b5de76bf8b6e74bd"}, - {file = "lxml-6.1.3-cp315-cp315t-manylinux_2_31_armv7l.whl", hash = "sha256:dc205732d593118cf701d986f40e9de7801bb2e371cb189ddbda9b7348f4d97e"}, - {file = "lxml-6.1.3-cp315-cp315t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:88e719b9437f148f7e1465df845c758dd1598618cbea3a2fd1e61a715542f2b2"}, - {file = "lxml-6.1.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:40983eabefd13da003e68170928c7acc011f0d095eefce5871a3c71c9385fb9a"}, - {file = "lxml-6.1.3-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:fad67b12ffe0f71e02b4932b04883cbc76a9072bbd30731409d3523cf058b011"}, - {file = "lxml-6.1.3-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6cd11e7550d89e551a87dcec30f04b1fca32e86b68708aa01a4daa455d8605e5"}, - {file = "lxml-6.1.3-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:ca0ec532ad2f5ba1e5ec120ac157769c57f01855b3d8bf37213f5d88abd9ba0a"}, - {file = "lxml-6.1.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:e99e09ab7741f1281e2677f4c0058c7f5267d182530b09c87e4f6aa26adf3887"}, - {file = "lxml-6.1.3-cp315-cp315t-win32.whl", hash = "sha256:ace1d2c83b2bd24db5940600541140e87a325e119cb32d5fa9ad720d7e76648e"}, - {file = "lxml-6.1.3-cp315-cp315t-win_amd64.whl", hash = "sha256:b49638355ea3bebba70da783ccbc630fd72afa16bc46c54474bfa1f9a915bbc6"}, - {file = "lxml-6.1.3-cp315-cp315t-win_arm64.whl", hash = "sha256:5a721a98c649855963811b59b55755b30566e7f7fc40bdc9803d66dee9f811cf"}, - {file = "lxml-6.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:13a620a3fcc20023f9e6ed5c383e00e826f1c2d5db554df2f67240760f9118e8"}, - {file = "lxml-6.1.3-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fbfb70ba01355251faf6b293171df49f73a88a1b6494db109ffea85442574458"}, - {file = "lxml-6.1.3-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:302f72413251c03f671e063c9414bed5dc8c927069e5abb69245521e51a4e81b"}, - {file = "lxml-6.1.3-cp38-cp38-manylinux_2_28_i686.whl", hash = "sha256:ce1f220114959941170e22b8ad44279f6dee2dcef7591814d01ae805dc058889"}, - {file = "lxml-6.1.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:170773d8a3cdc76259065523ddd978c44f9806e28605f08812e8f86783e44ac6"}, - {file = "lxml-6.1.3-cp38-cp38-win32.whl", hash = "sha256:92d96586376fb79a33474797186bf993250152ee5c32650b67db78d54b92e6f3"}, - {file = "lxml-6.1.3-cp38-cp38-win_amd64.whl", hash = "sha256:d44442effeb8781f392340c5dc8c6716fba41dbeacb82fd4c0f09026fb5ff682"}, - {file = "lxml-6.1.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:869dfcd4d381cb0ea87085cc4f011b9171b494ef21e76ad8665f6d5e2d1dc8a1"}, - {file = "lxml-6.1.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6ba4fe5bfbef6811a8e49b3719cde373ad399006c0c1ac184b7297116ecbba5d"}, - {file = "lxml-6.1.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:61116cec57ed69aebc70f37a545eec095339bb829efbdabcfb97c51e9536e158"}, - {file = "lxml-6.1.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e11e885e0704be185867fcf71b904d8f65d7d6877bc121f69870b0d0479ba7b"}, - {file = "lxml-6.1.3-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41e2d428110b408e963b6fb18f9bbf1f5c027b56bd4b498d54556476c0aeb1c3"}, - {file = "lxml-6.1.3-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aa9fd1ee2a5dacfc41039ed49ffeeacfa75bafbd255b69f3b578e11897a0e623"}, - {file = "lxml-6.1.3-cp39-cp39-manylinux_2_28_i686.whl", hash = "sha256:7f75b9b9fec2a9c6b18095c81865580e795b1441c429e42d22fcc82a77f40039"}, - {file = "lxml-6.1.3-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:cc669256d28736f7f3a149df5c380c50ace2692ba3e62203d10656fade4a2145"}, - {file = "lxml-6.1.3-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d077f21f4b16f0471353883748f126f62038760397c107bb9fad2ca94dc0dfb7"}, - {file = "lxml-6.1.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0d12846d6ce434fb3857918eef4315ec9b4769deb020c75828798614bfcfd"}, - {file = "lxml-6.1.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:2b9b1325ca1c2a9a2dbb6eb913ae563313f2082ae60b03210f7e83ee80712274"}, - {file = "lxml-6.1.3-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:a2e3f70673a1d5b82f38255f777d26cd855bf2092b1436c4867464a7892f9238"}, - {file = "lxml-6.1.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c34ca1dc41bd86d9ff830d5bdf4e4a752bba6c54f7d2707027ce0eabd36084c9"}, - {file = "lxml-6.1.3-cp39-cp39-win32.whl", hash = "sha256:b50343241eb69fd85f7791cf8bcc7b1c4729826b7d59ba2f6b27db29638fa745"}, - {file = "lxml-6.1.3-cp39-cp39-win_amd64.whl", hash = "sha256:0794e04ba343852c6d78e996c58ef4b8e579b4ecc72f8df0d4058bf843b4c96e"}, - {file = "lxml-6.1.3-cp39-cp39-win_arm64.whl", hash = "sha256:0ab2467e405e748d93495fb5568e74044802b8d3ff2b2a1607c3f78c6e982de5"}, - {file = "lxml-6.1.3-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4b061064b4a2fe8598a466d723d43dbcd5a610a5d5cfe02fb6226f5c17349f75"}, - {file = "lxml-6.1.3-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8499d464de86fab0f102313cce32a9bed9ab1f06ec813cf025cb790964fbb765"}, - {file = "lxml-6.1.3-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9e67324961ac9bbe616cce5100514d2e34d88665aeb07071e8b16eac55d06d94"}, - {file = "lxml-6.1.3-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5d12669a2c419b0e8dc423d23dea24bb82f6f9cb829f32e04674b0ba40322a7c"}, - {file = "lxml-6.1.3-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:97acecb11cbc411473f15b8d780df06d7a9f3a2aad9aca78364f56640c8fb70e"}, - {file = "lxml-6.1.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:f8b9c8ceebae6387d0dc77f7f4dbbfbfc962dba2efbfe6877486075a480726b4"}, - {file = "lxml-6.1.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d2765c18ce303149ee804b1f3dad11232726dd0a702d73a15cf19179ac8cc962"}, - {file = "lxml-6.1.3-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d5a748d12dd9b535e0a130f60dae9ddf0adafbabe61e7864f55c7436c84547a"}, - {file = "lxml-6.1.3-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:41096ec0740a58dad03d3ae0c7486d306d20becefb13ceb1649835ab3eb64167"}, - {file = "lxml-6.1.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:415e3a115c0d510e329020012834d1c0aa1c581ee53a218603e38abbc1dea70a"}, - {file = "lxml-6.1.3-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20428910dae17a1a93152a3ff2c0441d2f4932992c0797d65651dd0561f1792f"}, - {file = "lxml-6.1.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bc8dd3d9c93e70c3df974a201ac2958b6d77b465d813c51d1f15fa8e645763ae"}, - {file = "lxml-6.1.3-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3847e71a78cbbc1aff955dbbbaf2fff12153f611d3162c5beaa3395636cbc2f9"}, - {file = "lxml-6.1.3-pp39-pypy39_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe91993149523aa59941b9e3c90e2eb45f57ad014697aef6c8b13339a59c019e"}, - {file = "lxml-6.1.3-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71532ebf30be0048a45559b4fab15333fbaaf9042f658e878d918ecd0cf09805"}, - {file = "lxml-6.1.3-pp39-pypy39_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1b50797ac246bb2942a04b6c0f69af0667aba7cf7535f39bbb1b3208fd5d128"}, - {file = "lxml-6.1.3-pp39-pypy39_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2bb7d703bed7ac893bf7f40d97b5d9279d35d2ce460624ca28929eab0d5a3d"}, - {file = "lxml-6.1.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:be5346653c0b0e34be96869ff9dbeba23860156f89a2896a64c64fb419260cb6"}, - {file = "lxml-6.1.3.tar.gz", hash = "sha256:45222d94ddd511536f3b2f7d9deae3b2339b4ce0f075f1ca25703b07cad9dd21"}, -] - -[package.extras] -cssselect = ["cssselect (>=0.7)"] -html-clean = ["lxml_html_clean"] -html5 = ["html5lib"] -htmlsoup = ["BeautifulSoup4"] +dev = ["Sphinx (==8.1.3) ; python_version >= \"3.11\"", "build (==1.2.2) ; python_version >= \"3.11\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.5.0) ; python_version >= \"3.8\"", "mypy (==0.910) ; python_version < \"3.6\"", "mypy (==0.971) ; python_version == \"3.6\"", "mypy (==1.13.0) ; python_version >= \"3.8\"", "mypy (==1.4.1) ; python_version == \"3.7\"", "myst-parser (==4.0.0) ; python_version >= \"3.11\"", "pre-commit (==4.0.1) ; python_version >= \"3.9\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==8.3.2) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==5.0.0) ; python_version == \"3.8\"", "pytest-cov (==6.0.0) ; python_version >= \"3.9\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.1.0) ; python_version >= \"3.8\"", "sphinx-rtd-theme (==3.0.2) ; python_version >= \"3.11\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.23.2) ; python_version >= \"3.8\"", "twine (==6.0.1) ; python_version >= \"3.11\""] [[package]] name = "markdown" @@ -3660,7 +2806,7 @@ description = "Python implementation of John Gruber's Markdown." optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"eval\" or extra == \"docs\"" +markers = "extra == \"docs\"" files = [ {file = "markdown-3.10.2-py3-none-any.whl", hash = "sha256:e91464b71ae3ee7afd3017d9f358ef0baf158fd9a298db92f1d4761133824c36"}, {file = "markdown-3.10.2.tar.gz", hash = "sha256:994d51325d25ad8aa7ce4ebaec003febcce822c3f8c911e3b17c52f7f589f950"}, @@ -3674,16 +2820,16 @@ testing = ["coverage", "pyyaml"] name = "markdown-it-py" version = "2.2.0" description = "Python port of markdown-it. Markdown parsing, done right!" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] +markers = "extra == \"vllm\" or extra == \"tinker\" or extra == \"docs\"" files = [ {file = "markdown-it-py-2.2.0.tar.gz", hash = "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"}, {file = "markdown_it_py-2.2.0-py3-none-any.whl", hash = "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30"}, ] [package.dependencies] -linkify-it-py = {version = ">=1,<3", optional = true, markers = "extra == \"linkify\""} mdurl = ">=0.1,<1.0" [package.extras] @@ -3795,52 +2941,6 @@ files = [ {file = "markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698"}, ] -[[package]] -name = "math-verify" -version = "0.8.0" -description = "HuggingFace library for verifying mathematical answers" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "math_verify-0.8.0-py3-none-any.whl", hash = "sha256:31ca651296d817a9bb3fd58ca1fd0d192dcea709b1e5ecf2d0a4514c16f89087"}, - {file = "math_verify-0.8.0.tar.gz", hash = "sha256:3295e0adb94bfe553ff6e3189c44f1916a85aa24ab5d1900f2086a706e28f7c4"}, -] - -[package.dependencies] -latex2sympy2_extended = "1.10.2" - -[package.extras] -antlr4-11-0 = ["latex2sympy2_extended[antlr4-11-0]"] -antlr4-13-2 = ["latex2sympy2_extended[antlr4-13-2]"] -antlr4-9-3 = ["latex2sympy2_extended[antlr4-9-3]"] -dev = ["math-verify[format]", "math-verify[test]"] -format = ["ruff"] -inference = ["lighteval[math]"] -test = ["pytest"] - -[[package]] -name = "mcore-bridge" -version = "1.6.4" -description = "MCore-Bridge: Making Megatron training as simple as Transformers" -optional = true -python-versions = ">=3.8.0" -groups = ["main"] -markers = "extra == \"megatron\"" -files = [ - {file = "mcore_bridge-1.6.4-py3-none-any.whl", hash = "sha256:df8ab70f3a8122184833221af8f7756ff937bf980ee6fef6cbeaa3fcc1d00a73"}, - {file = "mcore_bridge-1.6.4.tar.gz", hash = "sha256:6df211f0e810d0d3d4be61de72bf68aa730c7da00a7ed5ecc18ffd1ec3563cf2"}, -] - -[package.dependencies] -megatron-core = ">=0.16,<0.20" -modelscope = "*" -peft = ">=0.11,<0.21" -safetensors = "*" -tqdm = "*" -transformers = ">=4.33,<5.17.0" - [[package]] name = "mcp" version = "1.26.0" @@ -3848,7 +2948,7 @@ description = "Model Context Protocol SDK" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "mcp-1.26.0-py3-none-any.whl", hash = "sha256:904a21c33c25aa98ddbeb47273033c435e595bbacfdb177f4bd87f6dceebe1ca"}, {file = "mcp-1.26.0.tar.gz", hash = "sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66"}, @@ -3882,7 +2982,7 @@ description = "Collection of plugins for markdown-it-py" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"client\" or extra == \"docs\"" +markers = "extra == \"docs\"" files = [ {file = "mdit-py-plugins-0.3.5.tar.gz", hash = "sha256:eee0adc7195e5827e17e02d2a258a2ba159944a0748f59c5099a4a27f78fcf6a"}, {file = "mdit_py_plugins-0.3.5-py3-none-any.whl", hash = "sha256:ca9a0714ea59a24b2b044a1831f48d817dd0c817e84339f20e7889f392d77c4e"}, @@ -3900,9 +3000,10 @@ testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] +markers = "extra == \"vllm\" or extra == \"tinker\" or extra == \"docs\"" files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -3945,7 +3046,7 @@ description = "Mistral-common is a library of common utilities for Mistral AI." optional = true python-versions = "<3.14,>=3.10.0" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "mistral_common-1.9.1-py3-none-any.whl", hash = "sha256:9e2b2520b6f67bac2e2bb06fcf985b7a1277b01938da2b7cda8cf0fdbfa92e91"}, {file = "mistral_common-1.9.1.tar.gz", hash = "sha256:550583d70a395c3586cfb748ffab53bd1d7c3409507f0efc0118bff30ffb26e9"}, @@ -4025,8 +3126,6 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.23.3", markers = "python_version >= \"3.11\""}, - {version = ">=1.21.2", markers = "python_version == \"3.10\""}, - {version = ">=2.1.0", markers = "python_version >= \"3.13\""}, {version = ">=1.26.0", markers = "python_version >= \"3.12\""}, ] @@ -4040,7 +3139,7 @@ description = "A framework for machine learning on Apple silicon." optional = true python-versions = ">=3.10" groups = ["main"] -markers = "(extra == \"rl\" or extra == \"async-rl\") and platform_machine == \"arm64\" and platform_system == \"Darwin\"" +markers = "extra == \"vllm\" and platform_machine == \"arm64\" and platform_system == \"Darwin\"" files = [ {file = "mlx-0.31.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:38680838e0dd9a621ed4adc5a9ed8b94aeb6a4798142fbe215b821b8c6b8fc36"}, {file = "mlx-0.31.0-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:7aded590bcf6839307c3acc899e196936991f97b499ddbdd0cd3b228bf10792f"}, @@ -4086,7 +3185,7 @@ description = "LLMs with MLX and the Hugging Face Hub" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "(extra == \"rl\" or extra == \"async-rl\") and platform_machine == \"arm64\" and platform_system == \"Darwin\"" +markers = "extra == \"vllm\" and platform_machine == \"arm64\" and platform_system == \"Darwin\"" files = [ {file = "mlx_lm-0.29.1-py3-none-any.whl", hash = "sha256:440941b3054c2a2216e97615de584cc90fa1ea874782e20699b9895721fad8dc"}, {file = "mlx_lm-0.29.1.tar.gz", hash = "sha256:b99180d8f33d33a077b814e550bfb2d8a59ae003d668fd1f4b3fff62a381d34b"}, @@ -4115,7 +3214,7 @@ description = "A framework for machine learning on Apple silicon." optional = true python-versions = ">=3.10" groups = ["main"] -markers = "(extra == \"rl\" or extra == \"async-rl\") and platform_machine == \"arm64\" and platform_system == \"Darwin\"" +markers = "extra == \"vllm\" and platform_machine == \"arm64\" and platform_system == \"Darwin\"" files = [ {file = "mlx_metal-0.31.0-py3-none-macosx_14_0_arm64.whl", hash = "sha256:1c572a6e3634a63060c103b0c38ac309e2d217be15519e3d8f0d6b452bb015f5"}, {file = "mlx_metal-0.31.0-py3-none-macosx_15_0_arm64.whl", hash = "sha256:554dc7cb29e0ea5fb6941df42f11a1de385b095848e6183c7a99d7c1f1a11f5d"}, @@ -4129,7 +3228,7 @@ description = "Python toolkit for standardized model hosting container implement optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "model_hosting_container_standards-0.1.13-py3-none-any.whl", hash = "sha256:be307d4a988cc660df4e6bd8bdedb7917844bac940e332f9fd001cb385d7994c"}, {file = "model_hosting_container_standards-0.1.13.tar.gz", hash = "sha256:27a1333410dde2719286a300a2803e24fdde407baa91894eb845c0f268aa194d"}, @@ -4157,47 +3256,19 @@ files = [ ] [package.dependencies] -addict = [ - {version = "*", optional = true, markers = "extra == \"framework\""}, - {version = "*", optional = true, markers = "extra == \"datasets\""}, -] -attrs = [ - {version = "*", optional = true, markers = "extra == \"framework\""}, - {version = "*", optional = true, markers = "extra == \"datasets\""}, -] -datasets = [ - {version = ">=3.0.0,<=3.6.0", optional = true, markers = "extra == \"framework\""}, - {version = ">=3.0.0,<=3.6.0", optional = true, markers = "extra == \"datasets\""}, -] -einops = [ - {version = "*", optional = true, markers = "extra == \"framework\""}, - {version = "*", optional = true, markers = "extra == \"datasets\""}, -] +addict = {version = "*", optional = true, markers = "extra == \"framework\""} +attrs = {version = "*", optional = true, markers = "extra == \"framework\""} +datasets = {version = ">=3.0.0,<=3.6.0", optional = true, markers = "extra == \"framework\""} +einops = {version = "*", optional = true, markers = "extra == \"framework\""} filelock = "*" -oss2 = {version = "*", optional = true, markers = "extra == \"datasets\""} -Pillow = [ - {version = "*", optional = true, markers = "extra == \"framework\""}, - {version = "*", optional = true, markers = "extra == \"datasets\""}, -] -python-dateutil = [ - {version = ">=2.1", optional = true, markers = "extra == \"framework\""}, - {version = ">=2.1", optional = true, markers = "extra == \"datasets\""}, -] +Pillow = {version = "*", optional = true, markers = "extra == \"framework\""} +python-dateutil = {version = ">=2.1", optional = true, markers = "extra == \"framework\""} PyYAML = {version = ">=5.4", optional = true, markers = "extra == \"framework\""} requests = ">=2.25" -scipy = [ - {version = "*", optional = true, markers = "extra == \"framework\""}, - {version = "*", optional = true, markers = "extra == \"datasets\""}, -] +scipy = {version = "*", optional = true, markers = "extra == \"framework\""} setuptools = "*" -simplejson = [ - {version = ">=3.3.0", optional = true, markers = "extra == \"framework\""}, - {version = ">=3.3.0", optional = true, markers = "extra == \"datasets\""}, -] -sortedcontainers = [ - {version = ">=1.5.9", optional = true, markers = "extra == \"framework\""}, - {version = ">=1.5.9", optional = true, markers = "extra == \"datasets\""}, -] +simplejson = {version = ">=3.3.0", optional = true, markers = "extra == \"framework\""} +sortedcontainers = {version = ">=1.5.9", optional = true, markers = "extra == \"framework\""} tqdm = ">=4.64.0" transformers = {version = "*", optional = true, markers = "extra == \"framework\""} urllib3 = ">=1.26" @@ -4221,19 +3292,6 @@ science = ["Pillow", "PyYAML (>=5.4)", "addict", "attrs", "biopython", "datasets server = ["fastapi", "sse-starlette", "uvicorn"] tests = ["expecttest", "flake8", "isort (>=4.3.21)", "pre-commit", "yapf (==0.30.0)"] -[[package]] -name = "more-itertools" -version = "11.1.0" -description = "More routines for operating on iterables, beyond itertools" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192"}, - {file = "more_itertools-11.1.0.tar.gz", hash = "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d"}, -] - [[package]] name = "mpmath" version = "1.3.0" @@ -4259,7 +3317,7 @@ description = "MessagePack serializer" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "msgpack-1.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2"}, {file = "msgpack-1.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87"}, @@ -4332,7 +3390,7 @@ description = "A fast serialization and validation library, with builtin support optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "msgspec-0.20.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:23a6ec2a3b5038c233b04740a545856a068bc5cb8db184ff493a58e08c994fbf"}, {file = "msgspec-0.20.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cde2c41ed3eaaef6146365cb0d69580078a19f974c6cb8165cc5dcd5734f573e"}, @@ -4553,9 +3611,6 @@ files = [ {file = "multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d"}, ] -[package.dependencies] -typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} - [[package]] name = "multiprocess" version = "0.70.16" @@ -4605,58 +3660,10 @@ sphinx = ">=5,<7" [package.extras] code-style = ["pre-commit (>=3.0,<4.0)"] linkify = ["linkify-it-py (>=1.0,<2.0)"] -rtd = ["ipython", "pydata-sphinx-theme (==v0.13.0rc4)", "sphinx-autodoc2 (>=0.4.2,<0.5.0)", "sphinx-book-theme (==1.0.0rc2)", "sphinx-copybutton", "sphinx-design2", "sphinx-pyscript", "sphinx-tippy (>=0.3.1)", "sphinx-togglebutton", "sphinxext-opengraph (>=0.7.5,<0.8.0)", "sphinxext-rediraffe (>=0.2.7,<0.3.0)"] +rtd = ["ipython", "pydata-sphinx-theme (==0.13.0rc4)", "sphinx-autodoc2 (>=0.4.2,<0.5.0)", "sphinx-book-theme (==1.0.0rc2)", "sphinx-copybutton", "sphinx-design2", "sphinx-pyscript", "sphinx-tippy (>=0.3.1)", "sphinx-togglebutton", "sphinxext-opengraph (>=0.7.5,<0.8.0)", "sphinxext-rediraffe (>=0.2.7,<0.3.0)"] testing = ["beautifulsoup4", "coverage[toml]", "pytest (>=7,<8)", "pytest-cov", "pytest-param-files (>=0.3.4,<0.4.0)", "pytest-regressions", "sphinx-pytest"] testing-docutils = ["pygments", "pytest (>=7,<8)", "pytest-param-files (>=0.3.4,<0.4.0)"] -[[package]] -name = "narwhals" -version = "2.26.0" -description = "Extremely lightweight compatibility layer between dataframe libraries" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "narwhals-2.26.0-py3-none-any.whl", hash = "sha256:29326d74f107c347fd1009bd58e38d9f7c7c5b51e6de97bc93dbc325d9038b54"}, - {file = "narwhals-2.26.0.tar.gz", hash = "sha256:6b9cadca82f375c7e4cf584fdc86ca25da54827307a9c58f94547ee6104b82dd"}, -] - -[package.extras] -cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] -dask = ["dask[dataframe] (>=2024.8)"] -duckdb = ["duckdb (>=1.1)"] -ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)"] -modin = ["modin (>=0.22.0)"] -pandas = ["pandas (>=1.3.4)"] -polars = ["polars (>=0.20.4)"] -pyarrow = ["pyarrow (>=13.0.0)"] -pyspark = ["pyspark (>=3.5.0)"] -pyspark-connect = ["pyspark[connect] (>=3.5.0)"] -sql = ["narwhals[duckdb]", "sqlparse (>=0.5.5)"] -sqlframe = ["sqlframe (>=3.22.0,!=3.39.3)"] - -[[package]] -name = "networkx" -version = "3.4.2" -description = "Python package for creating and manipulating graphs and networks" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "networkx-3.4.2-py3-none-any.whl", hash = "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f"}, - {file = "networkx-3.4.2.tar.gz", hash = "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1"}, -] - -[package.extras] -default = ["matplotlib (>=3.7)", "numpy (>=1.24)", "pandas (>=2.0)", "scipy (>=1.10,!=1.11.0,!=1.11.1)"] -developer = ["changelist (==0.5)", "mypy (>=1.1)", "pre-commit (>=3.2)", "rtoml"] -doc = ["intersphinx-registry", "myst-nb (>=1.1)", "numpydoc (>=1.8.0)", "pillow (>=9.4)", "pydata-sphinx-theme (>=0.15)", "sphinx (>=7.3)", "sphinx-gallery (>=0.16)", "texext (>=0.6.7)"] -example = ["cairocffi (>=1.7)", "contextily (>=1.6)", "igraph (>=0.11)", "momepy (>=0.7.2)", "osmnx (>=1.9)", "scikit-learn (>=1.5)", "seaborn (>=0.13)"] -extra = ["lxml (>=4.6)", "pydot (>=3.0.1)", "pygraphviz (>=1.14)", "sympy (>=1.10)"] -test = ["pytest (>=7.2)", "pytest-cov (>=4.0)"] - [[package]] name = "networkx" version = "3.6.1" @@ -4664,7 +3671,6 @@ description = "Python package for creating and manipulating graphs and networks" optional = false python-versions = "!=3.14.1,>=3.11" groups = ["main"] -markers = "python_version >= \"3.11\"" files = [ {file = "networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762"}, {file = "networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509"}, @@ -4688,7 +3694,7 @@ description = "Ninja is a small build system with a focus on speed" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "ninja-1.13.0-py3-none-macosx_10_9_universal2.whl", hash = "sha256:fa2a8bfc62e31b08f83127d1613d10821775a0eb334197154c4d6067b7068ff1"}, {file = "ninja-1.13.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3d00c692fb717fd511abeb44b8c5d00340c36938c12d6538ba989fe764e79630"}, @@ -4711,34 +3717,6 @@ files = [ {file = "ninja-1.13.0.tar.gz", hash = "sha256:4a40ce995ded54d9dc24f8ea37ff3bf62ad192b547f6c7126e7e25045e76f978"}, ] -[[package]] -name = "nltk" -version = "3.10.3" -description = "Natural Language Toolkit" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "nltk-3.10.3-py3-none-any.whl", hash = "sha256:ff9598a8e20518ee0d557745890cc4435b9578489e2dcbc69c4f81fa060caf7c"}, - {file = "nltk-3.10.3.tar.gz", hash = "sha256:bb9327a461c3811c2fa4900e03840401f2126adfb30c0072827c433bd2444ea4"}, -] - -[package.dependencies] -click = "*" -defusedxml = "*" -joblib = "*" -regex = ">=2021.8.3" -tqdm = "*" - -[package.extras] -all = ["matplotlib", "numpy", "pyparsing", "python-crfsuite", "requests", "scikit-learn", "scipy", "twython"] -corenlp = ["requests"] -machine-learning = ["numpy", "python-crfsuite", "scikit-learn", "scipy"] -plot = ["matplotlib"] -tgrep = ["pyparsing"] -twitter = ["twython"] - [[package]] name = "numba" version = "0.61.2" @@ -4746,7 +3724,7 @@ description = "compiling Python code using LLVM" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "numba-0.61.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:cf9f9fc00d6eca0c23fc840817ce9f439b9f03c8f03d6246c0e7f0cb15b7162a"}, {file = "numba-0.61.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ea0247617edcb5dd61f6106a56255baab031acc4257bddaeddb3a1003b4ca3fd"}, @@ -4939,7 +3917,7 @@ description = "CUDNN FrontEnd python library" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "nvidia_cudnn_frontend-1.18.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:baa6fbc8e7c55f1c78c0374ed9a890e1cf81acaca0c92d6135d18a8e3c985244"}, {file = "nvidia_cudnn_frontend-1.18.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e4bcca42259e358002c8867e3624a558f66cd5dff2cc6c3aafd860ef2f41730"}, @@ -5065,7 +4043,7 @@ description = "NVIDIA CUTLASS Python DSL" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "nvidia_cutlass_dsl-4.4.1-py3-none-any.whl", hash = "sha256:7b8ffa0117be35ef6c9a88f4462ee2a794efd0f7d9f65090e10a953e434fbfce"}, ] @@ -5083,7 +4061,7 @@ description = "NVIDIA CUTLASS Python DSL" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "nvidia_cutlass_dsl_libs_base-4.4.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:f8e3ecf646074bea34b301be283578806e02b712f277717f3dd1b28671aefa95"}, {file = "nvidia_cutlass_dsl_libs_base-4.4.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:fb2dc0039061fa4e03db6ffc54eaebc7a3b590f45463be461f048b819bb99efe"}, @@ -5107,7 +4085,7 @@ description = "Python Bindings for the NVIDIA Management Library" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" +markers = "extra == \"vllm\"" files = [ {file = "nvidia_ml_py-13.590.48-py3-none-any.whl", hash = "sha256:fd43d30ee9cd0b7940f5f9f9220b68d42722975e3992b6c21d14144c48760e43"}, {file = "nvidia_ml_py-13.590.48.tar.gz", hash = "sha256:8184d1be52914ac7f0991cd1c0d946c65dc88a840c754cd12c274b77b88760dd"}, @@ -5274,7 +4252,7 @@ description = "The official Python library for the openai API" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" +markers = "extra == \"vllm\"" files = [ {file = "openai-2.24.0-py3-none-any.whl", hash = "sha256:fed30480d7d6c884303287bde864980a4b137b60553ffbcf9ab4a233b7a73d94"}, {file = "openai-2.24.0.tar.gz", hash = "sha256:1e5769f540dbd01cb33bc4716a23e67b9d695161a734aff9c5f925e2bf99a673"}, @@ -5303,7 +4281,7 @@ description = "OpenAI's response format for its open-weight model series gpt-oss optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "openai_harmony-0.0.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:029ec25ca74abe48fdb58eb9fdd2a8c1618581fc33ce8e5653f8a1ffbfbd9326"}, {file = "openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4f709815924ec325b9a890e6ab2bbb0ceec8e319a4e257328eb752cf36b2efc"}, @@ -5333,7 +4311,7 @@ description = "A stats collection and distributed tracing framework" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "opencensus-0.11.4-py2.py3-none-any.whl", hash = "sha256:a18487ce68bc19900336e0ff4655c5a116daf10c1b3685ece8d971bddad6a864"}, {file = "opencensus-0.11.4.tar.gz", hash = "sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2"}, @@ -5351,7 +4329,7 @@ description = "OpenCensus Runtime Context" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "opencensus-context-0.1.3.tar.gz", hash = "sha256:a03108c3c10d8c80bb5ddf5c8a1f033161fa61972a9917f9b9b3a18517f0088c"}, {file = "opencensus_context-0.1.3-py2.py3-none-any.whl", hash = "sha256:073bb0590007af276853009fac7e4bab1d523c3f03baf4cb4511ca38967c6039"}, @@ -5364,7 +4342,7 @@ description = "Wrapper package for OpenCV python bindings." optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "opencv_python_headless-4.13.0.92-cp37-abi3-macosx_13_0_arm64.whl", hash = "sha256:1a7d040ac656c11b8c38677cc8cccdc149f98535089dbe5b081e80a4e5903209"}, {file = "opencv_python_headless-4.13.0.92-cp37-abi3-macosx_14_0_x86_64.whl", hash = "sha256:3e0a6f0a37994ec6ce5f59e936be21d5d6384a4556f2d2da9c2f9c5dc948394c"}, @@ -5386,7 +4364,7 @@ description = "OpenTelemetry Python API" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "(python_version >= \"3.11\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\") and (python_version == \"3.11\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\" or extra == \"client\") and (extra == \"client\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\")" +markers = "extra == \"ray\"" files = [ {file = "opentelemetry_api-1.39.1-py3-none-any.whl", hash = "sha256:2edd8463432a7f8443edce90972169b195e7d6a05500cd29e6d13898187c9950"}, {file = "opentelemetry_api-1.39.1.tar.gz", hash = "sha256:fbde8c80e1b937a2c61f20347e91c0c18a1940cecf012d62e65a7caf08967c9c"}, @@ -5396,92 +4374,6 @@ files = [ importlib-metadata = ">=6.0,<8.8.0" typing-extensions = ">=4.5.0" -[[package]] -name = "opentelemetry-exporter-otlp" -version = "1.39.1" -description = "OpenTelemetry Collector Exporters" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"server\"" -files = [ - {file = "opentelemetry_exporter_otlp-1.39.1-py3-none-any.whl", hash = "sha256:68ae69775291f04f000eb4b698ff16ff685fdebe5cb52871bc4e87938a7b00fe"}, - {file = "opentelemetry_exporter_otlp-1.39.1.tar.gz", hash = "sha256:7cf7470e9fd0060c8a38a23e4f695ac686c06a48ad97f8d4867bc9b420180b9c"}, -] - -[package.dependencies] -opentelemetry-exporter-otlp-proto-grpc = "1.39.1" -opentelemetry-exporter-otlp-proto-http = "1.39.1" - -[[package]] -name = "opentelemetry-exporter-otlp-proto-common" -version = "1.39.1" -description = "OpenTelemetry Protobuf encoding" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"server\"" -files = [ - {file = "opentelemetry_exporter_otlp_proto_common-1.39.1-py3-none-any.whl", hash = "sha256:08f8a5862d64cc3435105686d0216c1365dc5701f86844a8cd56597d0c764fde"}, - {file = "opentelemetry_exporter_otlp_proto_common-1.39.1.tar.gz", hash = "sha256:763370d4737a59741c89a67b50f9e39271639ee4afc999dadfe768541c027464"}, -] - -[package.dependencies] -opentelemetry-proto = "1.39.1" - -[[package]] -name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.39.1" -description = "OpenTelemetry Collector Protobuf over gRPC Exporter" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"server\"" -files = [ - {file = "opentelemetry_exporter_otlp_proto_grpc-1.39.1-py3-none-any.whl", hash = "sha256:fa1c136a05c7e9b4c09f739469cbdb927ea20b34088ab1d959a849b5cc589c18"}, - {file = "opentelemetry_exporter_otlp_proto_grpc-1.39.1.tar.gz", hash = "sha256:772eb1c9287485d625e4dbe9c879898e5253fea111d9181140f51291b5fec3ad"}, -] - -[package.dependencies] -googleapis-common-protos = ">=1.57,<2.0" -grpcio = [ - {version = ">=1.63.2,<2.0.0", markers = "python_version < \"3.13\""}, - {version = ">=1.66.2,<2.0.0", markers = "python_version >= \"3.13\""}, -] -opentelemetry-api = ">=1.15,<2.0" -opentelemetry-exporter-otlp-proto-common = "1.39.1" -opentelemetry-proto = "1.39.1" -opentelemetry-sdk = ">=1.39.1,<1.40.0" -typing-extensions = ">=4.6.0" - -[package.extras] -gcp-auth = ["opentelemetry-exporter-credential-provider-gcp (>=0.59b0)"] - -[[package]] -name = "opentelemetry-exporter-otlp-proto-http" -version = "1.39.1" -description = "OpenTelemetry Collector Protobuf over HTTP Exporter" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"server\"" -files = [ - {file = "opentelemetry_exporter_otlp_proto_http-1.39.1-py3-none-any.whl", hash = "sha256:d9f5207183dd752a412c4cd564ca8875ececba13be6e9c6c370ffb752fd59985"}, - {file = "opentelemetry_exporter_otlp_proto_http-1.39.1.tar.gz", hash = "sha256:31bdab9745c709ce90a49a0624c2bd445d31a28ba34275951a6a362d16a0b9cb"}, -] - -[package.dependencies] -googleapis-common-protos = ">=1.52,<2.0" -opentelemetry-api = ">=1.15,<2.0" -opentelemetry-exporter-otlp-proto-common = "1.39.1" -opentelemetry-proto = "1.39.1" -opentelemetry-sdk = ">=1.39.1,<1.40.0" -requests = ">=2.7,<3.0" -typing-extensions = ">=4.5.0" - -[package.extras] -gcp-auth = ["opentelemetry-exporter-credential-provider-gcp (>=0.59b0)"] - [[package]] name = "opentelemetry-exporter-prometheus" version = "0.60b1" @@ -5489,7 +4381,7 @@ description = "Prometheus Metric Exporter for OpenTelemetry" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "opentelemetry_exporter_prometheus-0.60b1-py3-none-any.whl", hash = "sha256:49f59178de4f4590e3cef0b8b95cf6e071aae70e1f060566df5546fad773b8fd"}, {file = "opentelemetry_exporter_prometheus-0.60b1.tar.gz", hash = "sha256:a4011b46906323f71724649d301b4dc188aaa068852e814f4df38cc76eac616b"}, @@ -5500,42 +4392,6 @@ opentelemetry-api = ">=1.12,<2.0" opentelemetry-sdk = ">=1.39.1,<1.40.0" prometheus-client = ">=0.5.0,<1.0.0" -[[package]] -name = "opentelemetry-instrumentation" -version = "0.60b1" -description = "Instrumentation Tools & Auto Instrumentation for OpenTelemetry Python" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"server\"" -files = [ - {file = "opentelemetry_instrumentation-0.60b1-py3-none-any.whl", hash = "sha256:04480db952b48fb1ed0073f822f0ee26012b7be7c3eac1a3793122737c78632d"}, - {file = "opentelemetry_instrumentation-0.60b1.tar.gz", hash = "sha256:57ddc7974c6eb35865af0426d1a17132b88b2ed8586897fee187fd5b8944bd6a"}, -] - -[package.dependencies] -opentelemetry-api = ">=1.4,<2.0" -opentelemetry-semantic-conventions = "0.60b1" -packaging = ">=18.0" -wrapt = ">=1.0.0,<2.0.0" - -[[package]] -name = "opentelemetry-instrumentation-logging" -version = "0.60b1" -description = "OpenTelemetry Logging instrumentation" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"server\"" -files = [ - {file = "opentelemetry_instrumentation_logging-0.60b1-py3-none-any.whl", hash = "sha256:f2e18cbc7e1dd3628c80e30d243897fdc93c5b7e0c8ae60abd2b9b6a99f82343"}, - {file = "opentelemetry_instrumentation_logging-0.60b1.tar.gz", hash = "sha256:98f4b9c7aeb9314a30feee7c002c7ea9abea07c90df5f97fb058b850bc45b89a"}, -] - -[package.dependencies] -opentelemetry-api = ">=1.12,<2.0" -opentelemetry-instrumentation = "0.60b1" - [[package]] name = "opentelemetry-proto" version = "1.39.1" @@ -5543,7 +4399,7 @@ description = "OpenTelemetry Python Proto" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" +markers = "extra == \"ray\"" files = [ {file = "opentelemetry_proto-1.39.1-py3-none-any.whl", hash = "sha256:22cdc78efd3b3765d09e68bfbd010d4fc254c9818afd0b6b423387d9dee46007"}, {file = "opentelemetry_proto-1.39.1.tar.gz", hash = "sha256:6c8e05144fc0d3ed4d22c2289c6b126e03bcd0e6a7da0f16cedd2e1c2772e2c8"}, @@ -5559,7 +4415,7 @@ description = "OpenTelemetry Python SDK" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" +markers = "extra == \"ray\"" files = [ {file = "opentelemetry_sdk-1.39.1-py3-none-any.whl", hash = "sha256:4d5482c478513ecb0a5d938dcc61394e647066e0cc2676bee9f3af3f3f45f01c"}, {file = "opentelemetry_sdk-1.39.1.tar.gz", hash = "sha256:cf4d4563caf7bff906c9f7967e2be22d0d6b349b908be0d90fb21c8e9c995cc6"}, @@ -5577,7 +4433,7 @@ description = "OpenTelemetry Semantic Conventions" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" +markers = "extra == \"ray\"" files = [ {file = "opentelemetry_semantic_conventions-0.60b1-py3-none-any.whl", hash = "sha256:9fa8c8b0c110da289809292b0591220d3a7b53c1526a23021e977d68597893fb"}, {file = "opentelemetry_semantic_conventions-0.60b1.tar.gz", hash = "sha256:87c228b5a0669b748c76d76df6c364c369c28f1c465e50f661e39737e84bc953"}, @@ -5587,90 +4443,13 @@ files = [ opentelemetry-api = "1.39.1" typing-extensions = ">=4.5.0" -[[package]] -name = "orjson" -version = "3.12.0" -description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "(python_version >= \"3.11\" or extra == \"async-rl\") and (extra == \"async-rl\" or extra == \"client\")" -files = [ - {file = "orjson-3.12.0-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:747843254519dd43b93eee3153a19e5a509334320c4d2f823ec879232db5c796"}, - {file = "orjson-3.12.0-cp310-cp310-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:7c2ad193c8004254f34b499f3bd2c80f043d10754aff2b38f93da574f4883f98"}, - {file = "orjson-3.12.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:bc7a872f03522d90e0429e6c0c5cd23084f767bedcb4c58048eec19294613344"}, - {file = "orjson-3.12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18a87929f31d94a77f7dc93cf527e91f39ce7fe7813d588a4de2507efd32a387"}, - {file = "orjson-3.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9683ee9ea0659da64f36574ef675b8a86330c34c19ea75db1fb93c3ff99e0ef"}, - {file = "orjson-3.12.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:103b5db66aa53c1f9e88c2524be4f383e831ba7dfd5f9f5af6336a177c622f11"}, - {file = "orjson-3.12.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bd57d79aefa3f84eec851d6de7a366795b9345cfaf17f82b4820430a7a5fa241"}, - {file = "orjson-3.12.0-cp310-cp310-win32.whl", hash = "sha256:3dbce9b6b3074b31a5d5dd322a9c4e5b16f206091ece4194c2e36952847a105e"}, - {file = "orjson-3.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:3bb17a06f9bd15237b3216c044209fe92597379124018cfc196fbb846cde64df"}, - {file = "orjson-3.12.0-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a94f0f0c6fcbb2b5bd9734c57a489c7584a732bbdf04a39e8c83b861e9d03e92"}, - {file = "orjson-3.12.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:a696529ec96a90d9a5f9570207efe403c8b08f8e4aa2783ee3403511e2fdfa10"}, - {file = "orjson-3.12.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:e4ac5059baab4b3acbd99485de019ff8cda0fdf34b61fa74f7197a53db78bfe8"}, - {file = "orjson-3.12.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:8e29957429c35bbb5a185a119c523aa2428b7bbf1a293724c7b9375ed8f892a3"}, - {file = "orjson-3.12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dce0166feb0a737ab84f598c9a338cbc0b764a036617aa686194f53c7eba0c3e"}, - {file = "orjson-3.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9caf3d09f47c3c70c4451ada20ef9bc4a4cdffa26f49862cf0a253b329aae2d5"}, - {file = "orjson-3.12.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b9dca132b1fda5565088e65a6b6e742285e0aeceb6fae549fa8863e16c7d3998"}, - {file = "orjson-3.12.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a791f793b287bbc135b8e87c34e35c8bfc693e2a8a620fab1ae682b925f9a32e"}, - {file = "orjson-3.12.0-cp311-cp311-win32.whl", hash = "sha256:31ed278a36304390adc3eec5d7f6fd593a7c3e99e5a06cd07866396c4b1b4710"}, - {file = "orjson-3.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb2539159dfe8d371914f354360fa50e4a577cc89222a3828b9650a5e5040252"}, - {file = "orjson-3.12.0-cp311-cp311-win_arm64.whl", hash = "sha256:61318b6de893c7a9d9f3e5ecbadccbfc26a7eb417ccc7bbf0771de3b4d72f868"}, - {file = "orjson-3.12.0-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:aa3e43a6846e91d7bde3d5a9c66090fcd8744f569a9b6cffc5e1ca38f6a461c0"}, - {file = "orjson-3.12.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:11edb4660a6680abee9788a3a9072208a2c96538cc1322bd79542065229d8e54"}, - {file = "orjson-3.12.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2d3a9da945a4d96ae758fdaaca56742e6b73b6fd554c5d8876f252a6dad70b83"}, - {file = "orjson-3.12.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:92ffc09e07233a6ab6d4e067f7841edcbcc134cb4812155cf171ea5255a421d7"}, - {file = "orjson-3.12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf44e374aadde77b1f6109f1030be51433eb61984379852766b6f4e187db7b1e"}, - {file = "orjson-3.12.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1192a7021b6d071aaf909864f6e924d6a2675ca360485b972b8401749311750b"}, - {file = "orjson-3.12.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:53c0c474a9d9aff9aebfc0c88de1f28f843d940e6e3a80729abdf6a20274356f"}, - {file = "orjson-3.12.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:532ff8cd4bd59a327a953a7dcde922c7fc25b85e29721bb8633265430d3a3873"}, - {file = "orjson-3.12.0-cp312-cp312-win32.whl", hash = "sha256:a6cf4b18e7de173f209f2084ffbd736dd72389a396326ee80a7022168be232e5"}, - {file = "orjson-3.12.0-cp312-cp312-win_amd64.whl", hash = "sha256:010811c1b69773450a01cef97727a67b223242f350b77d4ca000e59a9ef2155a"}, - {file = "orjson-3.12.0-cp312-cp312-win_arm64.whl", hash = "sha256:ad29eece0c601737f2a60edc2752a84e7a0785df3efb62e3012834700a5afe0d"}, - {file = "orjson-3.12.0-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9a36ec60f1796f9a3f13e3b98390295e17a1c7c10155b448d264098bf9ee5900"}, - {file = "orjson-3.12.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ad0422b92d5195443a39f80c3bcf731cc2e00f153bd32063a47b73b057bd0f03"}, - {file = "orjson-3.12.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:5a0fdbc216388f653d3752ff310e710f59253bd4ed6a2bfb3f4f06b84714bbd8"}, - {file = "orjson-3.12.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2eb5c56e534127b2b8fa38d2363c8b1b8190367ee0d1d16c041517d880843b94"}, - {file = "orjson-3.12.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:784106539f4b9d4b930e0b4eb8d45168507dae001945e71b4675a367f1e5e806"}, - {file = "orjson-3.12.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1c680706fc8396d95e7c4c1f9482563f552137aef91b57237a3ad5aaf64629df"}, - {file = "orjson-3.12.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:83445adc40cba26d6d621185a45128ce455b766af368cad2ab64b970603a7978"}, - {file = "orjson-3.12.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:644d005bc82f917337a95ce270c9f6f92f9834c2bed7b1477572f8db00784222"}, - {file = "orjson-3.12.0-cp313-cp313-win32.whl", hash = "sha256:d8e78d3d93705e3d27cc17cdb209e44d7a8ea203010cac6ce9c7ffc1ae1996f1"}, - {file = "orjson-3.12.0-cp313-cp313-win_amd64.whl", hash = "sha256:b85931be5b6763c31283805c9bdaae1ca03ad9f6f12a15f1cbf6745b907932c2"}, - {file = "orjson-3.12.0-cp313-cp313-win_arm64.whl", hash = "sha256:6a31348d7dfa64cd9c78bd1f510ff44c48fe64d71094e6b90e364dba3b55949e"}, - {file = "orjson-3.12.0-cp314-cp314-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9e6fee342a48760e854d743e7a81534d8e2925a6f46e09f750cf56b50fd1de5d"}, - {file = "orjson-3.12.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:8c3bb86dd10f39b3fbf434b7d5dc7cac77d6fc8ac572ae30a10731ede2c4b647"}, - {file = "orjson-3.12.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:2bb3ce43203936072dd8b4917b01d3aecfc02329bfb42510cb7cfb24708adc9c"}, - {file = "orjson-3.12.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:6a2a79c89984dc719817d388c8709e0efc2a2795a934eaa746b4882eb6045adc"}, - {file = "orjson-3.12.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f06dd838d1e07d9b1de0932ec0485ec92c4d5f5d1ad4817a656268c3e88be1e1"}, - {file = "orjson-3.12.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6b11be792c3d2c6a4be2af4ebf97a68d0bf5f580aca6e86a418a354f6cc846a"}, - {file = "orjson-3.12.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:477ecaf6b9f88f873341b91fcc736119ca81b5e002a9f7f308ff5b4f2ce2a70e"}, - {file = "orjson-3.12.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f3c0683136acdc29afdf88a5bc2f7d3d0e34087788d1d63c0144b805a87a196f"}, - {file = "orjson-3.12.0-cp314-cp314-win32.whl", hash = "sha256:d39f3f5c3927e2dc0913fe5bbc1a2f6b1b9d1bba1de6358340d0ad0d0c00ca92"}, - {file = "orjson-3.12.0-cp314-cp314-win_amd64.whl", hash = "sha256:0b1ac5bf6609b2716c7954011c5fef6254922df029f45d032ee4ebf5d363cbed"}, - {file = "orjson-3.12.0-cp314-cp314-win_arm64.whl", hash = "sha256:50fae885cb073eac7556353ff3df93312b0d5137b0a5056b2bb63f97ed9a93c7"}, - {file = "orjson-3.12.0-cp315-cp315-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:01efac2074fffb4cb1ea3fab7861e9d0f2a26913854a972f5ac760525dbdaf6e"}, - {file = "orjson-3.12.0-cp315-cp315-macosx_15_0_arm64.whl", hash = "sha256:ed4ca42bd55955aa34deedcfdfd0e0c31abf51143aae158ae2bc3520b626e517"}, - {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_aarch64.whl", hash = "sha256:40f92192227505acca4e2533ce565f8e6b9535f7d0d09b0968452f18b7376b38"}, - {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_armv7l.whl", hash = "sha256:33efefcf5d88eaf400b47e2eba02f91f319bb9951be61ca500b7d536d3f2079d"}, - {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_i686.whl", hash = "sha256:8e386b0bc0ddd7cd2056f884b5a0af33592bd01ac66a7ca4b42a65a7e7774a13"}, - {file = "orjson-3.12.0-cp315-cp315-manylinux_2_39_x86_64.whl", hash = "sha256:58c58e1de0006ffb580368d6793c36c7b0b021db066479cf281bf5061e732328"}, - {file = "orjson-3.12.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:08231552159be266a7269555bd9f7c016aee7d9ad6dab06eb58796c5ccb7101c"}, - {file = "orjson-3.12.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a15f9a891bce5f5cc5d210e3ad8614d4d1b489a56448c099d6d2a7168b2d954a"}, - {file = "orjson-3.12.0-cp315-cp315-win32.whl", hash = "sha256:03091c8a64db4be38746597ceea68f33c238e27acd9bfe99fb59420224ae7a55"}, - {file = "orjson-3.12.0-cp315-cp315-win_amd64.whl", hash = "sha256:2b7bcefb9f40fa242fa6b06377232c048e655747790829609168c01162f60578"}, - {file = "orjson-3.12.0-cp315-cp315-win_arm64.whl", hash = "sha256:859fc4196855890150bb08e649b30d2c93b249b3e3edd0d3bb2231abf8aa8adc"}, - {file = "orjson-3.12.0.tar.gz", hash = "sha256:d14203fb1aae2ad9b3d52f8a0e82aeb10197ef1c9bc61da7f358bd70b00123d5"}, -] - [[package]] name = "oss2" version = "2.13.1" description = "Aliyun OSS (Object Storage Service) SDK" -optional = true +optional = false python-versions = "*" groups = ["main"] -markers = "extra == \"eval\"" files = [ {file = "oss2-2.13.1.tar.gz", hash = "sha256:8548ea7d43326f6fd679bc8b79b3a2dfbfe9c6a60ed57e2410818fec57023dda"}, ] @@ -5690,7 +4469,7 @@ description = "Structured Text Generation in Rust" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "outlines_core-0.2.11-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:89d79d8454b321f60047541a896d410ca9db631d241960266c4fe839cf5cd1b1"}, {file = "outlines_core-0.2.11-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:44d581893f8644da02db7be11887229a40d26077cbdd22072ad1ed1db0ad0b2d"}, @@ -5738,19 +4517,6 @@ files = [ [package.extras] test = ["asv", "coverage[toml] (>=5.1)", "diff-cover", "maturin[patchelf]", "numba", "numpy", "pre-commit", "psutil", "pydantic", "pytest", "pytest-benchmark", "pytest-cov", "pytest-mock", "scipy", "torch"] -[[package]] -name = "overrides" -version = "7.7.0" -description = "A decorator to automatically detect mismatch when overriding a method." -optional = true -python-versions = ">=3.6" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49"}, - {file = "overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a"}, -] - [[package]] name = "packaging" version = "26.0" @@ -5763,103 +4529,6 @@ files = [ {file = "packaging-26.0.tar.gz", hash = "sha256:00243ae351a257117b6a241061796684b084ed1c516a08c48a3f7e147a9d80b4"}, ] -[[package]] -name = "pandas" -version = "2.3.3" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, - {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, - {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, - {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, - {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, - {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, - {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, - {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, - {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, - {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, - {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, - {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, - {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, - {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, - {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, - {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, - {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, - {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, - {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, - {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, - {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, - {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, - {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, - {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, - {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, - {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, - {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, - {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, - {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, - {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, - {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, - {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, -] - -[package.dependencies] -numpy = {version = ">=1.22.4", markers = "python_version < \"3.11\""} -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.7" - -[package.extras] -all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] -aws = ["s3fs (>=2022.11.0)"] -clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] -compression = ["zstandard (>=0.19.0)"] -computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] -feather = ["pyarrow (>=10.0.1)"] -fss = ["fsspec (>=2022.11.0)"] -gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] -hdf5 = ["tables (>=3.8.0)"] -html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] -mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] -parquet = ["pyarrow (>=10.0.1)"] -performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] -plot = ["matplotlib (>=3.6.3)"] -postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] -pyarrow = ["pyarrow (>=10.0.1)"] -spss = ["pyreadstat (>=1.2.0)"] -sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.9.2)"] - [[package]] name = "pandas" version = "3.0.1" @@ -5867,7 +4536,6 @@ description = "Powerful data structures for data analysis, time series, and stat optional = false python-versions = ">=3.11" groups = ["main"] -markers = "python_version >= \"3.11\"" files = [ {file = "pandas-3.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:de09668c1bf3b925c07e5762291602f0d789eca1b3a781f99c1c78f6cac0e7ea"}, {file = "pandas-3.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:24ba315ba3d6e5806063ac6eb717504e499ce30bd8c236d8693a5fd3f084c796"}, @@ -5957,7 +4625,7 @@ description = "Parse partial JSON generated by LLM" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "partial_json_parser-0.2.1.1.post7-py3-none-any.whl", hash = "sha256:145119e5eabcf80cbb13844a6b50a85c68bf99d376f8ed771e2a3c3b03e653ae"}, {file = "partial_json_parser-0.2.1.1.post7.tar.gz", hash = "sha256:86590e1ba6bcb6739a2dfc17d2323f028cb5884f4c6ce23db376999132c9a922"}, @@ -6112,117 +4780,12 @@ description = "A small Python package for determining appropriate platform-speci optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" +markers = "extra == \"ray\"" files = [ {file = "platformdirs-4.9.2-py3-none-any.whl", hash = "sha256:9170634f126f8efdae22fb58ae8a0eaa86f38365bc57897a6c4f781d1f5875bd"}, {file = "platformdirs-4.9.2.tar.gz", hash = "sha256:9a33809944b9db043ad67ca0db94b14bf452cc6aeaac46a88ea55b26e2e9d291"}, ] -[[package]] -name = "plotext" -version = "6.1.0" -description = "plotext plots directly on terminal" -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"client\"" -files = [ - {file = "plotext-6.1.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:4fec26b86ad32b3597de5861cfbf0023bb5ae02250f0ca940b12b069bccf43e6"}, - {file = "plotext-6.1.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:bfafad58a20da6c279e722481b5c3e67a9c8c54bde722d414c5ceffe71b7c652"}, - {file = "plotext-6.1.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c035440296be20e539ee70d4de07290d84bb0c9017bb3bc5867a8b3f9cead56"}, - {file = "plotext-6.1.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e59f54cedfb56129e2aec1f885869fe13903fdd4e6f18a10301e71a7d5b9851c"}, - {file = "plotext-6.1.0-py3-none-win_amd64.whl", hash = "sha256:60c85326da5454db482c9b1357c9ba385bae4a05c9dd498ff604e80458e03e3b"}, - {file = "plotext-6.1.0.tar.gz", hash = "sha256:6b729ce6376c5c1c2152fe85e0449dcc54879d0923b3c6c8d1670108245c6f84"}, -] - -[package.extras] -image = ["pillow (>=8.4)"] -video = ["ffpyplayer (>=4.3.5)", "pillow (>=8.4)", "yt-dlp (>=2024.1.1)"] - -[[package]] -name = "plotly" -version = "7.0.0" -description = "An open-source interactive data visualization library for Python" -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "plotly-7.0.0-py3-none-any.whl", hash = "sha256:78cbf7bd06d1b05bb3b8ec1b709864695229b55151b6f7530fbf55517ead6fdd"}, - {file = "plotly-7.0.0.tar.gz", hash = "sha256:08b21f1244a97e7a1a699833c4bb2678475aa108b3f1989886ed0b038ebfd849"}, -] - -[package.dependencies] -narwhals = ">=1.15.1" -packaging = "*" - -[package.extras] -dev = ["anywidget", "build", "colorcet", "fiona (<=1.9.6) ; python_version <= \"3.8\"", "geopandas", "inflect", "jupyter-builder", "jupyterlab", "kaleido (>=1.3.0)", "numpy (>=1.22)", "orjson", "pandas", "pdfrw", "pillow", "polars[timezone]", "pyarrow", "pytest", "pytz", "requests", "ruff (==0.11.12)", "scikit-image", "scipy", "statsmodels", "vaex ; python_version <= \"3.9\"", "xarray"] -dev-build = ["build", "jupyter-builder", "pytest", "requests", "ruff (==0.11.12)"] -dev-codegen = ["inflect", "pytest", "requests", "ruff (==0.11.12)"] -dev-core = ["pytest", "requests", "ruff (==0.11.12)"] -dev-optional = ["anywidget", "build", "colorcet", "fiona (<=1.9.6) ; python_version <= \"3.8\"", "geopandas", "inflect", "jupyter-builder", "jupyterlab", "kaleido (>=1.3.0)", "numpy (>=1.22)", "orjson", "pandas", "pdfrw", "pillow", "polars[timezone]", "pyarrow", "pytest", "pytz", "requests", "ruff (==0.11.12)", "scikit-image", "scipy", "statsmodels", "vaex ; python_version <= \"3.9\"", "xarray"] -dev-pandas1 = ["numpy (>=1,<2)", "pandas (>=1,<2)", "setuptools (<82)"] -dev-pandas2 = ["pandas (>=2,<3)"] -dev-pandas3 = ["pandas (>=3) ; python_version >= \"3.11\""] -express = ["numpy (>=1.22)"] -kaleido = ["kaleido (>=1.3.0)"] - -[[package]] -name = "pluggy" -version = "1.6.0" -description = "plugin and hook calling mechanisms for python" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"test\"" -files = [ - {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, - {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, -] - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["coverage", "pytest", "pytest-benchmark"] - -[[package]] -name = "portalocker" -version = "4.3.2" -description = "Cross-platform file locking, with Redis, PID-file and bounded-semaphore locks" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "portalocker-4.3.2-py3-none-any.whl", hash = "sha256:42f0362e7343af78b25eb095c0cde300e746a0ab6647a835cc30764918490c93"}, - {file = "portalocker-4.3.2.tar.gz", hash = "sha256:8de4a1330f55b3c9a1d50ab1b70a765f2f136d6399ae206411117ed98c3b8e2b"}, -] - -[package.extras] -docs = ["furo", "sphinx (>=7)"] -redis = ["redis (>=5.0)"] -tests = ["coverage-conditional-plugin (>=0.9)", "fakeredis (>=2.31.0)", "portalocker[redis]", "pytest (>=5.4.1)", "pytest-cov (>=2.8.1)", "pytest-rerunfailures (>=15.1)", "pytest-timeout (>=2.1.0)", "types-pywin32 (>=310.0.0.20250429)", "typing-extensions (>=4.4)"] -win32 = ["pywin32 (>=226) ; sys_platform == \"win32\""] - -[[package]] -name = "prettytable" -version = "3.18.0" -description = "A simple Python library for easily displaying tabular data in a visually appealing ASCII table format" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "prettytable-3.18.0-py3-none-any.whl", hash = "sha256:b3346e0e6f79180833aebaac088ae926340586cf6d7d991b9eb125b65f72313a"}, - {file = "prettytable-3.18.0.tar.gz", hash = "sha256:439217116152244369caf3d9f1caf2f9fe29b03bd79e88d2928c8e718c95d680"}, -] - -[package.dependencies] -wcwidth = ">=0.3.5" - -[package.extras] -tests = ["pytest (>=9)", "pytest-cov", "pytest-lazy-fixtures"] - [[package]] name = "prometheus-client" version = "0.24.1" @@ -6230,7 +4793,7 @@ description = "Python client for the Prometheus monitoring system." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "prometheus_client-0.24.1-py3-none-any.whl", hash = "sha256:150db128af71a5c2482b36e588fc8a6b95e498750da4b17065947c16070f4055"}, {file = "prometheus_client-0.24.1.tar.gz", hash = "sha256:7e0ced7fbbd40f7b84962d5d2ab6f17ef88a72504dcf7c0b40737b43b2a461f9"}, @@ -6248,7 +4811,7 @@ description = "Instrument your FastAPI app with Prometheus metrics" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "prometheus_fastapi_instrumentator-7.1.0-py3-none-any.whl", hash = "sha256:978130f3c0bb7b8ebcc90d35516a6fe13e02d2eb358c8f83887cdef7020c31e9"}, {file = "prometheus_fastapi_instrumentator-7.1.0.tar.gz", hash = "sha256:be7cd61eeea4e5912aeccb4261c6631b3f227d8924542d79eaf5af3f439cbe5e"}, @@ -6397,7 +4960,7 @@ description = "Beautiful, Pythonic protocol buffers" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "proto_plus-1.27.1-py3-none-any.whl", hash = "sha256:e4643061f3a4d0de092d62aa4ad09fa4756b2cbb89d4627f3985018216f9fefc"}, {file = "proto_plus-1.27.1.tar.gz", hash = "sha256:912a7460446625b792f6448bade9e55cd4e41e6ac10e27009ef71a7f317fa147"}, @@ -6416,7 +4979,7 @@ description = "" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "(python_version >= \"3.11\" or extra == \"async-rl\" or extra == \"rl\" or extra == \"server\" or extra == \"megatron\") and (extra == \"async-rl\" or extra == \"rl\" or extra == \"server\" or extra == \"megatron\" or extra == \"client\")" +markers = "extra == \"ray\" or extra == \"vllm\" or extra == \"megatron\"" files = [ {file = "protobuf-6.33.5-cp310-abi3-win32.whl", hash = "sha256:d71b040839446bac0f4d162e758bea99c8251161dae9d0983a3b88dee345153b"}, {file = "protobuf-6.33.5-cp310-abi3-win_amd64.whl", hash = "sha256:3093804752167bcab3998bec9f1048baae6e29505adaf1afd14a37bddede533c"}, @@ -6465,18 +5028,6 @@ files = [ dev = ["abi3audit", "black", "check-manifest", "colorama ; os_name == \"nt\"", "coverage", "packaging", "psleak", "pylint", "pyperf", "pypinfo", "pyreadline3 ; os_name == \"nt\"", "pytest", "pytest-cov", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "requests", "rstcheck", "ruff", "setuptools", "sphinx", "sphinx_rtd_theme", "toml-sort", "twine", "validate-pyproject[all]", "virtualenv", "vulture", "wheel", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] test = ["psleak", "pytest", "pytest-instafail", "pytest-xdist", "pywin32 ; os_name == \"nt\" and implementation_name != \"pypy\"", "setuptools", "wheel ; os_name == \"nt\" and implementation_name != \"pypy\"", "wmi ; os_name == \"nt\" and implementation_name != \"pypy\""] -[[package]] -name = "pwinput" -version = "1.0.3" -description = "A cross-platform Python module that displays **** for password input. Works on Windows, unlike getpass. Formerly called stdiomask." -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "pwinput-1.0.3.tar.gz", hash = "sha256:ca1a8bd06e28872d751dbd4132d8637127c25b408ea3a349377314a5491426f3"}, -] - [[package]] name = "py-cpuinfo" version = "9.0.0" @@ -6484,23 +5035,10 @@ description = "Get CPU info with pure Python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690"}, - {file = "py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5"}, -] - -[[package]] -name = "py-data-juicer" -version = "1.1.0" -description = "A One-Stop Data Processing System for Large Language Models." -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"data\"" -files = [ - {file = "py_data_juicer-1.1.0-py3-none-any.whl", hash = "sha256:27faf8ab1ee4a9597c09b4f9f18286a2292b1d7fb4ffe08b94ab5a1108489221"}, - {file = "py_data_juicer-1.1.0.tar.gz", hash = "sha256:675dca2b0fde9341c676351a29aec7cfa0a756608d64160a2745fe07e0fb4bf2"}, + {file = "py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5"}, ] [[package]] @@ -6510,7 +5048,7 @@ description = "" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "py_spy-0.4.1-py2.py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:809094208c6256c8f4ccadd31e9a513fe2429253f48e20066879239ba12cd8cc"}, {file = "py_spy-0.4.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:1fb8bf71ab8df95a95cc387deed6552934c50feef2cf6456bc06692a5508fd0c"}, @@ -6592,7 +5130,7 @@ description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "pyasn1-0.6.2-py3-none-any.whl", hash = "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf"}, {file = "pyasn1-0.6.2.tar.gz", hash = "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b"}, @@ -6605,7 +5143,7 @@ description = "A collection of ASN.1-based protocols modules" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, @@ -6621,7 +5159,7 @@ description = "Fast Base64 encoding/decoding" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "pybase64-1.4.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f63aa7f29139b8a05ce5f97cdb7fad63d29071e5bdc8a638a343311fe996112a"}, {file = "pybase64-1.4.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f5943ec1ae87a8b4fe310905bb57205ea4330c75e2c628433a7d9dd52295b588"}, @@ -6847,7 +5385,7 @@ description = "ISO country, subdivision, language, currency and script definitio optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "pycountry-26.2.16-py3-none-any.whl", hash = "sha256:115c4baf7cceaa30f59a4694d79483c9167dbce7a9de4d3d571c5f3ea77c305a"}, {file = "pycountry-26.2.16.tar.gz", hash = "sha256:5b6027d453fcd6060112b951dd010f01f168b51b4bf8a1f1fc8c95c8d94a0801"}, @@ -6857,10 +5395,10 @@ files = [ name = "pycparser" version = "3.0" description = "C parser in Python" -optional = false +optional = true python-versions = ">=3.10" groups = ["main"] -markers = "(platform_python_implementation != \"PyPy\" or implementation_name == \"pypy\") and (extra == \"rl\" or extra == \"async-rl\" or implementation_name == \"pypy\") and implementation_name != \"PyPy\"" +markers = "platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\" and (extra == \"vllm\" or extra == \"ray\") or extra == \"vllm\" and implementation_name == \"pypy\"" files = [ {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, @@ -6870,10 +5408,9 @@ files = [ name = "pycryptodome" version = "3.23.0" description = "Cryptographic library for Python" -optional = true +optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" groups = ["main"] -markers = "extra == \"eval\"" files = [ {file = "pycryptodome-3.23.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a176b79c49af27d7f6c12e4b178b0824626f40a7b9fed08f712291b6d54bf566"}, {file = "pycryptodome-3.23.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:573a0b3017e06f2cffd27d92ef22e46aa3be87a2d317a5abf7cc0e84e321bd75"}, @@ -7082,7 +5619,7 @@ description = "Extra Pydantic types." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "pydantic_extra_types-2.11.0-py3-none-any.whl", hash = "sha256:84b864d250a0fc62535b7ec591e36f2c5b4d1325fa0017eb8cda9aeb63b374a6"}, {file = "pydantic_extra_types-2.11.0.tar.gz", hash = "sha256:4e9991959d045b75feb775683437a97991d02c138e00b59176571db9ce634f0e"}, @@ -7109,7 +5646,7 @@ description = "Settings management using Pydantic" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "pydantic_settings-2.13.1-py3-none-any.whl", hash = "sha256:d56fd801823dbeae7f0975e1f8c8e25c258eb75d278ea7abb5d9cebb01b56237"}, {file = "pydantic_settings-2.13.1.tar.gz", hash = "sha256:b4c11847b15237fb0171e1462bf540e294affb9b86db4d9aa5c01730bdbe4025"}, @@ -7157,37 +5694,14 @@ doc = ["ablog (>=0.11.8)", "colorama", "graphviz", "ipykernel", "ipyleaflet", "i i18n = ["Babel", "jinja2"] test = ["pytest", "pytest-cov", "pytest-regressions", "sphinx[test]"] -[[package]] -name = "pyecharts" -version = "2.1.0" -description = "Python options, make charting easier" -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "pyecharts-2.1.0-py3-none-any.whl", hash = "sha256:5cba0aa06af3d9390a1fed18aa4e76acd7bdfb323cdf131874f46eaf7e78c6c7"}, - {file = "pyecharts-2.1.0.tar.gz", hash = "sha256:077f8205390aea705b8cc0be9d0c09916a2c060e9e5c8d9c9d11b7d8f4bdc1da"}, -] - -[package.dependencies] -jinja2 = "*" -prettytable = "*" -simplejson = "*" - -[package.extras] -images = ["PIL"] -phantomjs = ["snapshot-phantomjs"] -pyppeteer = ["snapshot-pyppeteer"] -selenium = ["snapshot-selenium"] - [[package]] name = "pygments" version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] +markers = "extra == \"vllm\" or extra == \"tinker\" or extra == \"docs\"" files = [ {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, @@ -7203,7 +5717,7 @@ description = "JSON Web Token implementation in Python" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "pyjwt-2.11.0-py3-none-any.whl", hash = "sha256:94a6bde30eb5c8e04fee991062b534071fd1439ef58d2adc9ccb823e7bcd0469"}, {file = "pyjwt-2.11.0.tar.gz", hash = "sha256:35f95c1f0fbe5d5ba6e43f00271c275f7a1a4db1dab27bf708073b75318ea623"}, @@ -7218,237 +5732,6 @@ dev = ["coverage[toml] (==7.10.7)", "cryptography (>=3.4.0)", "pre-commit", "pyt docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==7.10.7)", "pytest (>=8.4.2,<9.0.0)"] -[[package]] -name = "pylatexenc" -version = "2.11" -description = "Simple LaTeX parser providing latex-to-unicode and unicode-to-latex conversion" -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "pylatexenc-2.11-py2.py3-none-any.whl", hash = "sha256:e78e7391d6c104f1ed150e21cfaa58016cdb50aa54406a2eecb793649ffdfdd0"}, - {file = "pylatexenc-2.11.tar.gz", hash = "sha256:305a072a99ce736246049c9da05841b9d718c0f7ea8888f5f596cf15cb621053"}, -] - -[[package]] -name = "pynvml" -version = "13.0.1" -description = "Python utilities for the NVIDIA Management Library" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"server\"" -files = [ - {file = "pynvml-13.0.1-py3-none-any.whl", hash = "sha256:e2b20e0a501eeec951e2455b7ab444759cf048e0e13a57b08049fa2775266aa8"}, - {file = "pynvml-13.0.1.tar.gz", hash = "sha256:1245991d9db786b4d2f277ce66869bd58f38ac654e38c9397d18f243c8f6e48f"}, -] - -[package.dependencies] -nvidia-ml-py = ">=12.0.0" - -[package.extras] -test = ["pytest (>=3.6)", "pytest-cov", "pytest-runner"] - -[[package]] -name = "pyqwest" -version = "0.10.0" -description = "A modern, high-performance HTTP client for Python and Rust." -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version >= \"3.11\" and extra == \"client\"" -files = [ - {file = "pyqwest-0.10.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:aa492d5777dd145a60795ed95d9d4707a3cd1091fdcdfc93a82ac7fdc43ebacd"}, - {file = "pyqwest-0.10.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:59f3f16628e518c674102e7b5fcff2101bba6abb4f6737ec5fade9b9278e6a53"}, - {file = "pyqwest-0.10.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6e7db305a8318b1f3218053e87501f8f245ca8bd63e948e0282d04bf0883470"}, - {file = "pyqwest-0.10.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a5c757cfac5f53c8671dcb4850d5fc4c4339ea3e90636331c9318f8e3ddabc06"}, - {file = "pyqwest-0.10.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:234b3f71e3f314d997c203d8cf829b7117edd041153f9c277d0060ab90134148"}, - {file = "pyqwest-0.10.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5637256a0dac0ef57e0eaa02b032014965e4a4c995e1deca1b1b97e6d1765f78"}, - {file = "pyqwest-0.10.0-cp310-abi3-win_amd64.whl", hash = "sha256:7ea761937acf3a00d1a7e70e982949d18946e5471d1419266ab3a78bbfa19759"}, - {file = "pyqwest-0.10.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a21f1f15252a8303623b4f17b9c6de595ace11b3ade07f2adb6d07121e8191aa"}, - {file = "pyqwest-0.10.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:eb472c6e5d6833ebfec79db310e426eb17b01ac64c0e2c251bd9192c0d2ee0c5"}, - {file = "pyqwest-0.10.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83578e24cccd5e0dc04d60a0af7bfb43325b5f22d03ff74ff79ed0ecf553b50d"}, - {file = "pyqwest-0.10.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bb511c434f79c641efb5573e5795e56dc972252f4b96e52a9636d4ece5231a4"}, - {file = "pyqwest-0.10.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1aaccd8a9db9430b2aedb5bad8ead80742cbc056b85c229516c70dc80539f906"}, - {file = "pyqwest-0.10.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:73d9eb438ab4a957a1ce0619d3af8c1c1126bfb9181033b123d792fcf4224531"}, - {file = "pyqwest-0.10.0-cp312-cp312-win_amd64.whl", hash = "sha256:317a74d633abe3bc5bccabf479e069c515dab9e6a755274b0ccb1d8a5bbfede3"}, - {file = "pyqwest-0.10.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3978e794b9cfd8eaa500fb5d7aee63bc6172c605efa0abc1f62d85485bc049e1"}, - {file = "pyqwest-0.10.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:715991fd4f04862cd7a9d7452daabcdbd74dff4dff55eb20c22d60382dc2a4ed"}, - {file = "pyqwest-0.10.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c04798bed79c1dfa0e5b0e30fb137124311083490d44d6dfbe068d3dd254349e"}, - {file = "pyqwest-0.10.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35b472877e73dd63fed089c2bc8fa198407f005c8c19e0a93f025ebefde01a81"}, - {file = "pyqwest-0.10.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:564ec360b7848b35e009038ffbca00466305a9708ab21829477f64aa8cad4c64"}, - {file = "pyqwest-0.10.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b5c80e88a5967c1cadb3237c450f91a84a3683f8838c8dca96f09fee3612e762"}, - {file = "pyqwest-0.10.0-cp313-cp313-win_amd64.whl", hash = "sha256:dc3d80b402fb59dbe015e25993ac8147456fb231a4c949f92a89f31315ad50f9"}, - {file = "pyqwest-0.10.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:23a28beb55fa6d975949bffae4adfb69378f3229bb5cbd71231e95bf66f5b26c"}, - {file = "pyqwest-0.10.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e4415ae40b8eedb1713dab14d7f9fecc3f79d26f3206c561087b88b99d5ce24b"}, - {file = "pyqwest-0.10.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14b875d2273212d7fa8e4b755d8d736ffd226b1c707a9c0017dfdc8393a96eca"}, - {file = "pyqwest-0.10.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5071491e416ea54e3b95bf9ffbed0bd065b093cb96e10a75c3d8f2cbe3c9823"}, - {file = "pyqwest-0.10.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b68b5e68d513a4c63a072f8f40e38015160cf90bfbf7e8ef7c3935ca87e9e022"}, - {file = "pyqwest-0.10.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c48910d27820b9c46fcd001b0fe514a3cf47d4784f59512dcdb8c91c395f82e4"}, - {file = "pyqwest-0.10.0-cp314-cp314-win_amd64.whl", hash = "sha256:d03ba2cd17948b623a6210981d342eb122546d8a8e910ec77511aff4b1acdd00"}, - {file = "pyqwest-0.10.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:07a0eb595f4096232c2d22549b6e4612c1ecada7934e46462c2c37ce14a89cfb"}, - {file = "pyqwest-0.10.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:26401baf7dafc71c8d12d2e8389519d141e6f7c14094d0dd4cf9ec1d3b5555bd"}, - {file = "pyqwest-0.10.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e5e3c436e041d8873ce5bb0fdcf9f9e86f5604e8f0ef9e03149efebd8cb474f6"}, - {file = "pyqwest-0.10.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:09364115761579eabfc79d1e954cdb3ded508dac1903fac7285d4c6f058c683f"}, - {file = "pyqwest-0.10.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:559674a98a8b1217e1830ecd41c9905bf2b60983c6b8017063dfac199f00727c"}, - {file = "pyqwest-0.10.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f399a696392fff3db3eef0a18ef65b8a3b8396d129193487d966b8eb11006376"}, - {file = "pyqwest-0.10.0-cp314-cp314t-win_amd64.whl", hash = "sha256:0f9163d6dd991bf1bf27308ba38ba021af660b15fffa47ebca98e41cf6f00309"}, - {file = "pyqwest-0.10.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:cfcc7ba0229baa17831582befb046ace167b368140dae022d0b89b8d586ba12c"}, - {file = "pyqwest-0.10.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:eff9ccf427604d34c635954def07b6113d4754f968073eef2df40bdb80b05bf5"}, - {file = "pyqwest-0.10.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78662158093f9d5c742368f4dd9956aa595f44c6ac0860777c98853ddd5e1610"}, - {file = "pyqwest-0.10.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0879a3f0b37876372aa328b9fee956165db174c1ab72464146fe515f49399ddb"}, - {file = "pyqwest-0.10.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:850b8de6ade09a60bdb2f969871a177c2c304b594b2034ba3f5962c7bea75551"}, - {file = "pyqwest-0.10.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:399802647ea646c6ac9b5460e541b7c209b7a13563c667b2690ded2060185f2e"}, - {file = "pyqwest-0.10.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c26f3de1feb5d066d7a66802a47407a93ba043696064ad80beda4a0a4bf10056"}, - {file = "pyqwest-0.10.0.tar.gz", hash = "sha256:6c1a693be17d57d2c2eca4085e32c2809c53090c16719a907c90ebcf1f40dc01"}, -] - -[package.dependencies] -opentelemetry-api = ">=1.39.1" - -[[package]] -name = "pyroaring" -version = "1.0.3" -description = "Library for handling efficiently sorted integer sets." -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "pyroaring-1.0.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:c10e4cfbe203a578c78808406af491e3615d5e46cf69a7709050243346cd68bc"}, - {file = "pyroaring-1.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cc329c62e504f2531c4008240f31736bcd2dee4339071f1eac0648068e6d17fa"}, - {file = "pyroaring-1.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c7fb6ddf6ef31148f0939bc5c26b681d63df301ee1e372525012dd7bfe4a30a"}, - {file = "pyroaring-1.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd18446832ea04a7d33bd6b78270b0be14eabcda5937af3428d6cb3d2bf98e54"}, - {file = "pyroaring-1.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f0cbc766df2a24e28f23d69b66bbec64e691799219fd82c2f2236f03fc88e2e"}, - {file = "pyroaring-1.0.3-cp310-cp310-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:96a51e96f8f473381615f0f852f7238ad0a47f28e4a35e9f082468c5cfe4e9c3"}, - {file = "pyroaring-1.0.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:654af38b1f1c9bdc27b4f6d331fc5d91599df96e72a6df1886f4d95eea60ab29"}, - {file = "pyroaring-1.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:6721036afa31c07bdcbb4fcafa166660cf9c2eac695dcd495f8778549fa55899"}, - {file = "pyroaring-1.0.3-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:0caa10f20329d09233fac6550b2adce4d9f173f748a9a9a5ea3b7033827dfe2d"}, - {file = "pyroaring-1.0.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:f109be8af937e85c52cb920d3fd120db52b172f59460852d2e3d2e3d13a4f52a"}, - {file = "pyroaring-1.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ddc80bfcd313c7c524a2742d263e73cae088b6a611b77dcc46fa90c306f6dace"}, - {file = "pyroaring-1.0.3-cp310-cp310-win32.whl", hash = "sha256:5a183f5ec069757fe5b60e37f7c6fa8a53178eacf0d76601b739e2890edee036"}, - {file = "pyroaring-1.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:051bd9a66ce855a1143faa2b879ea6c6ca2905209e172ce9eedf79834897c730"}, - {file = "pyroaring-1.0.3-cp310-cp310-win_arm64.whl", hash = "sha256:3043ff5c85375310ca3cd3e01944e03026e0ec07885e52dfabcfcd9dc303867f"}, - {file = "pyroaring-1.0.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:755cdac1f9a1b7b5c621e570d4f6dbcf3b8e4a1e35a66f976104ecb35dce4ed2"}, - {file = "pyroaring-1.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ebab073db620f26f0ba11e13fa2f35e3b1298209fba47b6bc8cb6f0e2c9627f9"}, - {file = "pyroaring-1.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:684fb8dffe19bdb7f91897c65eac6eee23b1e46043c47eb24288f28a1170fe04"}, - {file = "pyroaring-1.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:678d31fc24e82945a1bfb14816c77823983382ffea76985d494782aa2f058427"}, - {file = "pyroaring-1.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d815f624e0285db3669f673d1725cb754b120ec70d0032d7c7166103a96c96d"}, - {file = "pyroaring-1.0.3-cp311-cp311-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:57fd5b80dacb8e888402b6b7508a734c6a527063e4e24e882ff2e0fd90721ada"}, - {file = "pyroaring-1.0.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab26a7a45a0bb46c00394d1a60a9f2d57c220f84586e30d59b39784b0f94aee6"}, - {file = "pyroaring-1.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:9232f3f606315d59049c128154100fd05008d5c5c211e48b21848cd41ee64d26"}, - {file = "pyroaring-1.0.3-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f34b44b3ec3df97b978799f2901fefb2a48d367496fd1cde3cc5fe8b3bc13510"}, - {file = "pyroaring-1.0.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:25a83ec6bac3106568bd3fdd316f0fee52aa0be8c72da565ad02b10ae7905924"}, - {file = "pyroaring-1.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c17d4ec53b5b6b333d9a9515051213a691293ada785dc8c025d3641482597ed3"}, - {file = "pyroaring-1.0.3-cp311-cp311-win32.whl", hash = "sha256:d54024459ace600f1d1ffbc6dc3c60eb47cca3b678701f06148f59e10f6f8d7b"}, - {file = "pyroaring-1.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:c28750148ef579a7447a8cb60b39e5943e03f8c29bce8f2788728f6f23d1887a"}, - {file = "pyroaring-1.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:535d8deccbd8db2c6bf38629243e9646756905574a742b2a72ff51d6461d616c"}, - {file = "pyroaring-1.0.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:add3e4c78eb590a76526ecce8d1566eecdd5822e351c36b3697997f4a80ed808"}, - {file = "pyroaring-1.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ebaffe846cf4ba4f00ce6b8a9f39613f24e2d09447e77be4fa6e898bc36451b6"}, - {file = "pyroaring-1.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a9459f27498f97d08031a34a5ead230b77eb0ab3cc3d85b7f54faa2fd548acd6"}, - {file = "pyroaring-1.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2b2eb8bd1c35c772994889be9f7dda09477475d7aa1e2af9ab4ef18619326f6"}, - {file = "pyroaring-1.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d31f4c1c906f1af14ce61a3959d04a14a64c594f8a768399146a45bbd341f21f"}, - {file = "pyroaring-1.0.3-cp312-cp312-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:53be988fc86698d56c11049bfe5113a2f6990adb1fa2782b29636509808b6aa7"}, - {file = "pyroaring-1.0.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7df84d223424523b19a23781f4246cc247fd6d821e1bc0853c2f25669136f7d0"}, - {file = "pyroaring-1.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:34a781f1f9766897f63ef18be129827340ae37764015b83fdcff1efb9e29136d"}, - {file = "pyroaring-1.0.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:1f414343b4ed0756734328cdf2a91022fc54503769e3f8d79bd0b672ea815a16"}, - {file = "pyroaring-1.0.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:d16ae185c72dc64f76335dbe53e53a892e78115adc92194957d1b7ef74d230b9"}, - {file = "pyroaring-1.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f888447bf22dde7759108bfe6dfbeb6bbb61b14948de9c4cb6843c4dd57e2215"}, - {file = "pyroaring-1.0.3-cp312-cp312-win32.whl", hash = "sha256:fbbdc44c51a0a3efd7be3dbe04466278ce098fcd101aa1905849319042159770"}, - {file = "pyroaring-1.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:3b217c4b3ad953b4c759a0d2f9bd95316f0c345b9f7adb49e6ded7a1f5106bd4"}, - {file = "pyroaring-1.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:e6bcf838564c21bab8fe6c2748b4990d4cd90612d8c470c04889def7bb5114ea"}, - {file = "pyroaring-1.0.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:20bc947054b197d1baa76cd05d70b8e04f95b82e698266e2f8f2f4b36d764477"}, - {file = "pyroaring-1.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ba5909b4c66bb85cab345e2f3a87e5ce671509c94b8c9823d8db64e107cbe854"}, - {file = "pyroaring-1.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b744746ba5da27fad760067f12633f5d384db6a1e65648d00244ceacbbd87731"}, - {file = "pyroaring-1.0.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5b16c2a2791a5a09c4b59c0e1069ac1c877d0df25cae3155579c7eac8844676e"}, - {file = "pyroaring-1.0.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7f68dfcf8d01177267f4bc06c4960fe8e39577470d1b52c9af8b61a72ca8767"}, - {file = "pyroaring-1.0.3-cp313-cp313-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:dba4e4700030182a981a3c887aa73887697145fc9ffb192f908aa59b718fbbdd"}, - {file = "pyroaring-1.0.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e26dd1dc1edba02288902914bdb559e53e346e9155defa43c31fcab831b55342"}, - {file = "pyroaring-1.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6eb98d2cacfc6d51c6a69893f04075e07b3df761eac71ba162c43b9b4c4452ad"}, - {file = "pyroaring-1.0.3-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a967e9eddb9485cbdd95d6371e3dada67880844d836c0283d3b11efe9225d1b7"}, - {file = "pyroaring-1.0.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b12ef7f992ba7be865f91c7c098fd8ac6c413563aaa14d5b1e2bcb8cb43a4614"}, - {file = "pyroaring-1.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:82ca5be174b85c40be7b00bc6bf39b2931a1b4a465f3af17ec6b9c48e9aa6fe0"}, - {file = "pyroaring-1.0.3-cp313-cp313-win32.whl", hash = "sha256:f758c681e63ffe74b20423695e71f0410920f41b075cee679ffb5bc2bf38440b"}, - {file = "pyroaring-1.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:428c3bb384fe4c483feb5cf7aa3aef1621fb0a5c4f3d391da67b2c4a43f08a10"}, - {file = "pyroaring-1.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:9c0c856e8aa5606e8aed5f30201286e404fdc9093f81fefe82d2e79e67472bb2"}, - {file = "pyroaring-1.0.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6321a95b5b2ba69aa32e920dd1aa7f8fc4fac55b75981978aa4f2378724dee27"}, - {file = "pyroaring-1.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:054eb6ef04ff9d2ed3ddd18ae21e5e51e02d0f8cdd7e5cb948648f77ddb04ea2"}, - {file = "pyroaring-1.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4d064aea3827e63eb60294ae3e6623e29613f5c8844869646d06f3735a425dd9"}, - {file = "pyroaring-1.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c84d5b17ef628c3956d9a79c2f78c5bea7dda6f7aeb01f34671034d2650b9efb"}, - {file = "pyroaring-1.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8038f7dd25eb83c277b8e0ea14c5e61f085cc76bd0c6b9f6679f1770e33541ec"}, - {file = "pyroaring-1.0.3-cp38-cp38-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:defc508ef7acaf58d07e603c55feda6742c4034f5262cfd616f92cc3adbc2815"}, - {file = "pyroaring-1.0.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dd0831326971b0ffa08ccce79abe7c2450d5d9254804d855e23a8ba31f70351a"}, - {file = "pyroaring-1.0.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:03e063329481396cbb70f1ce8b8ca0f01d74a45ee9d908b6645b0282b23832b0"}, - {file = "pyroaring-1.0.3-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:7a1b1c82d2da0bedc7c22d4047bd62544ef0e25c6be86ccf4b9d1ccc38876ee8"}, - {file = "pyroaring-1.0.3-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:dd7f9e5b7366b8f9bafca2a0fcf83fa534a00cc12d4ca01e301d8662bcdb805c"}, - {file = "pyroaring-1.0.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:a5a1db84e0952805223a7bf77eae58384b700a6b9affc53fb9772dddf868c712"}, - {file = "pyroaring-1.0.3-cp38-cp38-win32.whl", hash = "sha256:54cb0c2bddd330e22099773c4681aca90847265afe56a9201a92c1a758494261"}, - {file = "pyroaring-1.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:47d985293f861df1f2b03b41cef4fd3249c1c9608081750bcf3153051c2312d0"}, - {file = "pyroaring-1.0.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d46eb5db78b673d8d8ca83651a1cce1e15eec5a922f2951b1f61014463b72af5"}, - {file = "pyroaring-1.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ce202452de2b58bffa3eb02e27c681eefcfb54e27f8ef85b5c93ebaada50f3f3"}, - {file = "pyroaring-1.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:531b6ae56989b61742dde1b64fedc5537acc046cf04a333548322366c1bf3922"}, - {file = "pyroaring-1.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3035db9459bd8635a0145b4a9e3102869d621cb0b3648051115f06d31ffd1976"}, - {file = "pyroaring-1.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7c8fb6b0ad0e8db1b9559b2da180b103b48adddf0e4f24404269e2a3b5db268d"}, - {file = "pyroaring-1.0.3-cp39-cp39-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8d5df95d9511bc83048da9348c7ab1c20f97ff4d95faf27ee1fdf2e8a96e200e"}, - {file = "pyroaring-1.0.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:65d2d81e5aed7698fab23058db70fb2b65fad221090be037a0af498569109915"}, - {file = "pyroaring-1.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:e195636034a0b62ec0e5325ed2f610f39cc8955ace3f47a5bc7f484159f02341"}, - {file = "pyroaring-1.0.3-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:bb7f2561e3ec26c3c869458431cbcba6b83f7e925b024460c136dbb5fadf3b31"}, - {file = "pyroaring-1.0.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:8e996939de01f448eb9448d91b47ab60bff0555c2a80d5c12a8405814072cd35"}, - {file = "pyroaring-1.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c656d62d0cf96ede0edc4e7d392889238777bdf88b32afd5d51c3cab016c29a0"}, - {file = "pyroaring-1.0.3-cp39-cp39-win32.whl", hash = "sha256:a7a7d14822c64841ae64e98309697e1631ebadba55ded33daa7cd16d1b487d11"}, - {file = "pyroaring-1.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:a86b88adbe0531b75f94f87279a6d4ee68e63335e29bbdab4400a05704fc2587"}, - {file = "pyroaring-1.0.3-cp39-cp39-win_arm64.whl", hash = "sha256:1ed2e9c7af46052466b5fa0392fe540331474718d97b9756cefa23233bfdb3ea"}, - {file = "pyroaring-1.0.3.tar.gz", hash = "sha256:cd7392d1c010c9e41c11c62cd0610c8852e7e9698b1f7f6c2fcdefe50e7ef6da"}, -] - -[[package]] -name = "pytest" -version = "9.1.1" -description = "pytest: simple powerful testing with Python" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"test\"" -files = [ - {file = "pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c"}, - {file = "pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313"}, -] - -[package.dependencies] -colorama = {version = ">=0.4", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1", markers = "python_version < \"3.11\""} -iniconfig = ">=1.0.1" -packaging = ">=22" -pluggy = ">=1.5,<2" -pygments = ">=2.7.2" -tomli = {version = ">=1", markers = "python_version < \"3.11\""} - -[package.extras] -dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "requests", "setuptools", "xmlschema"] - -[[package]] -name = "pytest-asyncio" -version = "1.4.0" -description = "Pytest support for asyncio" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"test\"" -files = [ - {file = "pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1"}, - {file = "pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42"}, -] - -[package.dependencies] -backports-asyncio-runner = {version = ">=1.1,<2", markers = "python_version < \"3.11\""} -pytest = ">=8.4,<10" -typing-extensions = {version = ">=4.12", markers = "python_version < \"3.13\""} - -[package.extras] -docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1)", "sphinx-tabs (>=3.5)"] -testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] - [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -7471,7 +5754,7 @@ description = "Python interpreter discovery" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "python_discovery-1.1.0-py3-none-any.whl", hash = "sha256:a162893b8809727f54594a99ad2179d2ede4bf953e12d4c7abc3cc9cdbd1437b"}, {file = "python_discovery-1.1.0.tar.gz", hash = "sha256:447941ba1aed8cc2ab7ee3cb91be5fc137c5bdbb05b7e6ea62fbdcb66e50b268"}, @@ -7492,7 +5775,7 @@ description = "Read key-value pairs from a .env file and set them as environment optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a"}, {file = "python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3"}, @@ -7508,7 +5791,7 @@ description = "JSON Log Formatter for the Python Logging Package" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "python_json_logger-4.0.0-py3-none-any.whl", hash = "sha256:af09c9daf6a813aa4cc7180395f50f2a9e5fa056034c9953aec92e381c5ba1e2"}, {file = "python_json_logger-4.0.0.tar.gz", hash = "sha256:f58e68eb46e1faed27e0f574a55a0455eecd7b8a5b88b85a784519ba3cff047f"}, @@ -7524,38 +5807,12 @@ description = "A streaming multipart parser for Python" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155"}, {file = "python_multipart-0.0.22.tar.gz", hash = "sha256:7340bef99a7e0032613f56dc36027b959fd3b30a787ed62d310e951f7c3a3a58"}, ] -[[package]] -name = "pytz" -version = "2026.3.post1" -description = "World timezone definitions, modern and historical" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815"}, - {file = "pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d"}, -] - -[[package]] -name = "pyvers" -version = "0.2.3" -description = "A Python library for managing multiple versions of dependencies" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "pyvers-0.2.3-py3-none-any.whl", hash = "sha256:6f5b5612f2f4bd08caa49baa70fc5f875fc7da701a5385c13e244eea6b8114dd"}, - {file = "pyvers-0.2.3.tar.gz", hash = "sha256:c4b81c3a033963245e124cdecb052783c9c4cea3bb08c051833af1c44faa6283"}, -] - [[package]] name = "pywin32" version = "311" @@ -7563,7 +5820,7 @@ description = "Python for Window Extensions" optional = true python-versions = "*" groups = ["main"] -markers = "sys_platform == \"win32\" and (extra == \"rl\" or extra == \"async-rl\")" +markers = "extra == \"vllm\" and sys_platform == \"win32\"" files = [ {file = "pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3"}, {file = "pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b"}, @@ -7674,9 +5931,10 @@ files = [ name = "pyzmq" version = "27.1.0" description = "Python bindings for 0MQ" -optional = false +optional = true python-versions = ">=3.8" groups = ["main"] +markers = "extra == \"vllm\"" files = [ {file = "pyzmq-27.1.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:508e23ec9bc44c0005c4946ea013d9317ae00ac67778bd47519fdf5a0e930ff4"}, {file = "pyzmq-27.1.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:507b6f430bdcf0ee48c0d30e734ea89ce5567fd7b8a0f0044a369c176aa44556"}, @@ -7782,7 +6040,7 @@ description = "Ray provides a simple, universal API for building distributed app optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "ray-2.54.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:a22937f09ee74a43171df338d84b45ef882c1c05748947ca9d5343a44d4b9379"}, {file = "ray-2.54.0-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:1e63e491155695d527513ffe9d33a6aeb3f3cdccb6309adadfd6f8dd7c0300f7"}, @@ -7802,71 +6060,34 @@ files = [ ] [package.dependencies] -aiohttp = [ - {version = ">=3.13.3", optional = true, markers = "extra == \"serve\""}, - {version = ">=3.13.3", optional = true, markers = "extra == \"default\""}, -] -aiohttp_cors = [ - {version = "*", optional = true, markers = "extra == \"serve\""}, - {version = "*", optional = true, markers = "extra == \"default\""}, -] +aiohttp = {version = ">=3.13.3", optional = true, markers = "extra == \"serve\""} +aiohttp_cors = {version = "*", optional = true, markers = "extra == \"serve\""} click = ">=7.0" -colorful = [ - {version = "*", optional = true, markers = "extra == \"serve\""}, - {version = "*", optional = true, markers = "extra == \"default\""}, -] +colorful = {version = "*", optional = true, markers = "extra == \"serve\""} cupy-cuda12x = {version = "*", optional = true, markers = "sys_platform != \"darwin\" and extra == \"cgraph\""} fastapi = {version = "*", optional = true, markers = "extra == \"serve\""} filelock = "*" -grpcio = [ - {version = ">=1.42.0", optional = true, markers = "extra == \"serve\""}, - {version = ">=1.42.0", optional = true, markers = "extra == \"default\""}, -] +grpcio = {version = ">=1.42.0", optional = true, markers = "extra == \"serve\""} jsonschema = "*" msgpack = ">=1.0.0,<2.0.0" -opencensus = [ - {version = "*", optional = true, markers = "extra == \"serve\""}, - {version = "*", optional = true, markers = "extra == \"default\""}, -] -opentelemetry-exporter-prometheus = [ - {version = "*", optional = true, markers = "extra == \"serve\""}, - {version = "*", optional = true, markers = "extra == \"default\""}, -] -opentelemetry-proto = [ - {version = "*", optional = true, markers = "extra == \"serve\""}, - {version = "*", optional = true, markers = "extra == \"default\""}, -] -opentelemetry-sdk = [ - {version = ">=1.30.0", optional = true, markers = "extra == \"serve\""}, - {version = ">=1.30.0", optional = true, markers = "extra == \"default\""}, -] +opencensus = {version = "*", optional = true, markers = "extra == \"serve\""} +opentelemetry-exporter-prometheus = {version = "*", optional = true, markers = "extra == \"serve\""} +opentelemetry-proto = {version = "*", optional = true, markers = "extra == \"serve\""} +opentelemetry-sdk = {version = ">=1.30.0", optional = true, markers = "extra == \"serve\""} packaging = ">=24.2" -prometheus_client = [ - {version = ">=0.7.1", optional = true, markers = "extra == \"serve\""}, - {version = ">=0.7.1", optional = true, markers = "extra == \"default\""}, -] +prometheus_client = {version = ">=0.7.1", optional = true, markers = "extra == \"serve\""} protobuf = ">=3.20.3" py-spy = [ {version = ">=0.2.0", optional = true, markers = "python_version < \"3.12\" and extra == \"serve\""}, - {version = ">=0.2.0", optional = true, markers = "python_version < \"3.12\" and extra == \"default\""}, {version = ">=0.4.0", optional = true, markers = "python_version >= \"3.12\" and extra == \"serve\""}, ] -pydantic = [ - {version = "<2.0.dev0 || >=2.12.dev0,<3", optional = true, markers = "extra == \"serve\""}, - {version = "<2.0.dev0 || >=2.12.dev0,<3", optional = true, markers = "extra == \"default\""}, -] +pydantic = {version = "<2.0.dev0 || >=2.12.dev0,<3", optional = true, markers = "extra == \"serve\""} pyyaml = "*" requests = "*" -smart_open = [ - {version = "*", optional = true, markers = "extra == \"serve\""}, - {version = "*", optional = true, markers = "extra == \"default\""}, -] +smart_open = {version = "*", optional = true, markers = "extra == \"serve\""} starlette = {version = "*", optional = true, markers = "extra == \"serve\""} uvicorn = {version = "*", extras = ["standard"], optional = true, markers = "extra == \"serve\""} -virtualenv = [ - {version = ">=20.0.24,<20.21.1 || >20.21.1", optional = true, markers = "extra == \"serve\""}, - {version = ">=20.0.24,<20.21.1 || >20.21.1", optional = true, markers = "extra == \"default\""}, -] +virtualenv = {version = ">=20.0.24,<20.21.1 || >20.21.1", optional = true, markers = "extra == \"serve\""} watchfiles = {version = "*", optional = true, markers = "extra == \"serve\""} [package.extras] @@ -7906,30 +6127,6 @@ commonmark = ">=0.8.1" docutils = ">=0.11" sphinx = ">=1.3.1" -[[package]] -name = "redis" -version = "8.1.0" -description = "Python client for Redis database and key-value store" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"server\"" -files = [ - {file = "redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb"}, - {file = "redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25"}, -] - -[package.dependencies] -async-timeout = {version = ">=4.0.3", markers = "python_full_version < \"3.11.3\""} - -[package.extras] -circuit-breaker = ["pybreaker (>=1.4.0)"] -hiredis = ["hiredis (>=3.2.0)"] -jwt = ["pyjwt (>=2.13.0)"] -ocsp = ["cryptography (>=36.0.1)", "pyopenssl (>=20.0.1)", "requests (>=2.31.0)"] -otel = ["opentelemetry-api (>=1.39.1)", "opentelemetry-exporter-otlp-proto-http (>=1.39.1)", "opentelemetry-sdk (>=1.39.1)"] -xxhash = ["xxhash (>=3.6.0,<3.7.0)"] - [[package]] name = "referencing" version = "0.37.0" @@ -7937,7 +6134,7 @@ description = "JSON Referencing + Python" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231"}, {file = "referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8"}, @@ -8098,9 +6295,10 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rich" version = "14.3.3" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -optional = false +optional = true python-versions = ">=3.8.0" groups = ["main"] +markers = "extra == \"vllm\" or extra == \"tinker\"" files = [ {file = "rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d"}, {file = "rich-14.3.3.tar.gz", hash = "sha256:b8daa0b9e4eef54dd8cf7c86c03713f53241884e814f4e2f5fb342fe520f639b"}, @@ -8120,7 +6318,7 @@ description = "Rich toolkit for building command-line applications" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "rich_toolkit-0.19.7-py3-none-any.whl", hash = "sha256:0288e9203728c47c5a4eb60fd2f0692d9df7455a65901ab6f898437a2ba5989d"}, {file = "rich_toolkit-0.19.7.tar.gz", hash = "sha256:133c0915872da91d4c25d85342d5ec1dfacc69b63448af1a08a0d4b4f23ef46e"}, @@ -8138,7 +6336,7 @@ description = "Python Bindings for the ignore crate" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "rignore-0.7.6-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:f3c74a7e5ee77aea669c95fdb3933f2a6c7549893700082e759128a29cf67e45"}, {file = "rignore-0.7.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b7202404958f5fe3474bac91f65350f0b1dde1a5e05089f2946549b7e91e79ec"}, @@ -8296,40 +6494,6 @@ files = [ {file = "rignore-0.7.6.tar.gz", hash = "sha256:00d3546cd793c30cb17921ce674d2c8f3a4b00501cb0e3dd0e82217dbeba2671"}, ] -[[package]] -name = "rouge-chinese" -version = "1.0.3" -description = "Python ROUGE Score Implementation for Chinese Language Task (official rouge score)" -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "rouge_chinese-1.0.3-py3-none-any.whl", hash = "sha256:afda22cb64f9a0f1d7a19e88d7cd9acfd2c1f5d45bf463ade7ecadf87f30624d"}, - {file = "rouge_chinese-1.0.3.tar.gz", hash = "sha256:4074de21b01a83ca6032fe6d6354c5f618a305da60c994294133be2abf030f4a"}, -] - -[package.dependencies] -six = "*" - -[[package]] -name = "rouge-score" -version = "0.1.2" -description = "Pure python implementation of ROUGE-1.5.5." -optional = true -python-versions = ">=3.7" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "rouge_score-0.1.2.tar.gz", hash = "sha256:c7d4da2683e68c9abf0135ef915d63a46643666f848e558a1b9f7ead17ff0f04"}, -] - -[package.dependencies] -absl-py = "*" -nltk = "*" -numpy = "*" -six = ">=1.14.0" - [[package]] name = "rpds-py" version = "0.30.0" @@ -8337,7 +6501,7 @@ description = "Python bindings to Rust's persistent data structures (rpds)" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288"}, {file = "rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00"}, @@ -8463,7 +6627,7 @@ description = "Pure-Python RSA implementation" optional = true python-versions = "<4,>=3.6" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "rsa-4.9.1-py3-none-any.whl", hash = "sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762"}, {file = "rsa-4.9.1.tar.gz", hash = "sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75"}, @@ -8472,32 +6636,6 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" -[[package]] -name = "sacrebleu" -version = "2.6.0" -description = "Hassle-free computation of shareable, comparable, and reproducible BLEU, chrF, and TER scores" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "sacrebleu-2.6.0-py3-none-any.whl", hash = "sha256:3edc1531575cfe4ad04ce53491a9307e234af1c3f805a1f491cbec844229a8a8"}, - {file = "sacrebleu-2.6.0.tar.gz", hash = "sha256:91499b6cd46138d95154fff1e863c2f9be57e82f0c719d8dd718d0006cf6c566"}, -] - -[package.dependencies] -colorama = "*" -lxml = "*" -numpy = ">=1.17" -portalocker = "*" -regex = "*" -tabulate = ">=0.8.9" - -[package.extras] -dev = ["lxml-stubs", "mypy", "pytest", "setuptools", "types-tabulate", "wheel"] -ja = ["ipadic (>=1.0,<2.0)", "mecab-python3 (>=1.0.9,<2.0.0)"] -ko = ["mecab-ko (>=1.0.2,<2.0.0)", "mecab-ko-dic (>=1.0,<2.0)"] - [[package]] name = "safetensors" version = "0.7.0" @@ -8545,71 +6683,6 @@ testing = ["h5py (>=3.7.0)", "huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2 testingfree = ["huggingface-hub (>=0.12.1)", "hypothesis (>=6.70.2)", "pytest (>=7.2.0)", "pytest-benchmark (>=4.0.0)", "safetensors[numpy]", "setuptools-rust (>=1.5.2)"] torch = ["packaging", "safetensors[numpy]", "torch (>=1.10)"] -[[package]] -name = "scipy" -version = "1.15.3" -description = "Fundamental algorithms for scientific computing in Python" -optional = false -python-versions = ">=3.10" -groups = ["main"] -markers = "python_version == \"3.10\"" -files = [ - {file = "scipy-1.15.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:a345928c86d535060c9c2b25e71e87c39ab2f22fc96e9636bd74d1dbf9de448c"}, - {file = "scipy-1.15.3-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:ad3432cb0f9ed87477a8d97f03b763fd1d57709f1bbde3c9369b1dff5503b253"}, - {file = "scipy-1.15.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:aef683a9ae6eb00728a542b796f52a5477b78252edede72b8327a886ab63293f"}, - {file = "scipy-1.15.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:1c832e1bd78dea67d5c16f786681b28dd695a8cb1fb90af2e27580d3d0967e92"}, - {file = "scipy-1.15.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:263961f658ce2165bbd7b99fa5135195c3a12d9bef045345016b8b50c315cb82"}, - {file = "scipy-1.15.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2abc762b0811e09a0d3258abee2d98e0c703eee49464ce0069590846f31d40"}, - {file = "scipy-1.15.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:ed7284b21a7a0c8f1b6e5977ac05396c0d008b89e05498c8b7e8f4a1423bba0e"}, - {file = "scipy-1.15.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5380741e53df2c566f4d234b100a484b420af85deb39ea35a1cc1be84ff53a5c"}, - {file = "scipy-1.15.3-cp310-cp310-win_amd64.whl", hash = "sha256:9d61e97b186a57350f6d6fd72640f9e99d5a4a2b8fbf4b9ee9a841eab327dc13"}, - {file = "scipy-1.15.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:993439ce220d25e3696d1b23b233dd010169b62f6456488567e830654ee37a6b"}, - {file = "scipy-1.15.3-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:34716e281f181a02341ddeaad584205bd2fd3c242063bd3423d61ac259ca7eba"}, - {file = "scipy-1.15.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3b0334816afb8b91dab859281b1b9786934392aa3d527cd847e41bb6f45bee65"}, - {file = "scipy-1.15.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:6db907c7368e3092e24919b5e31c76998b0ce1684d51a90943cb0ed1b4ffd6c1"}, - {file = "scipy-1.15.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:721d6b4ef5dc82ca8968c25b111e307083d7ca9091bc38163fb89243e85e3889"}, - {file = "scipy-1.15.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39cb9c62e471b1bb3750066ecc3a3f3052b37751c7c3dfd0fd7e48900ed52982"}, - {file = "scipy-1.15.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:795c46999bae845966368a3c013e0e00947932d68e235702b5c3f6ea799aa8c9"}, - {file = "scipy-1.15.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:18aaacb735ab38b38db42cb01f6b92a2d0d4b6aabefeb07f02849e47f8fb3594"}, - {file = "scipy-1.15.3-cp311-cp311-win_amd64.whl", hash = "sha256:ae48a786a28412d744c62fd7816a4118ef97e5be0bee968ce8f0a2fba7acf3bb"}, - {file = "scipy-1.15.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6ac6310fdbfb7aa6612408bd2f07295bcbd3fda00d2d702178434751fe48e019"}, - {file = "scipy-1.15.3-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:185cd3d6d05ca4b44a8f1595af87f9c372bb6acf9c808e99aa3e9aa03bd98cf6"}, - {file = "scipy-1.15.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:05dc6abcd105e1a29f95eada46d4a3f251743cfd7d3ae8ddb4088047f24ea477"}, - {file = "scipy-1.15.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:06efcba926324df1696931a57a176c80848ccd67ce6ad020c810736bfd58eb1c"}, - {file = "scipy-1.15.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05045d8b9bfd807ee1b9f38761993297b10b245f012b11b13b91ba8945f7e45"}, - {file = "scipy-1.15.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:271e3713e645149ea5ea3e97b57fdab61ce61333f97cfae392c28ba786f9bb49"}, - {file = "scipy-1.15.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6cfd56fc1a8e53f6e89ba3a7a7251f7396412d655bca2aa5611c8ec9a6784a1e"}, - {file = "scipy-1.15.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0ff17c0bb1cb32952c09217d8d1eed9b53d1463e5f1dd6052c7857f83127d539"}, - {file = "scipy-1.15.3-cp312-cp312-win_amd64.whl", hash = "sha256:52092bc0472cfd17df49ff17e70624345efece4e1a12b23783a1ac59a1b728ed"}, - {file = "scipy-1.15.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2c620736bcc334782e24d173c0fdbb7590a0a436d2fdf39310a8902505008759"}, - {file = "scipy-1.15.3-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:7e11270a000969409d37ed399585ee530b9ef6aa99d50c019de4cb01e8e54e62"}, - {file = "scipy-1.15.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8c9ed3ba2c8a2ce098163a9bdb26f891746d02136995df25227a20e71c396ebb"}, - {file = "scipy-1.15.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0bdd905264c0c9cfa74a4772cdb2070171790381a5c4d312c973382fc6eaf730"}, - {file = "scipy-1.15.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79167bba085c31f38603e11a267d862957cbb3ce018d8b38f79ac043bc92d825"}, - {file = "scipy-1.15.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9deabd6d547aee2c9a81dee6cc96c6d7e9a9b1953f74850c179f91fdc729cb7"}, - {file = "scipy-1.15.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:dde4fc32993071ac0c7dd2d82569e544f0bdaff66269cb475e0f369adad13f11"}, - {file = "scipy-1.15.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f77f853d584e72e874d87357ad70f44b437331507d1c311457bed8ed2b956126"}, - {file = "scipy-1.15.3-cp313-cp313-win_amd64.whl", hash = "sha256:b90ab29d0c37ec9bf55424c064312930ca5f4bde15ee8619ee44e69319aab163"}, - {file = "scipy-1.15.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3ac07623267feb3ae308487c260ac684b32ea35fd81e12845039952f558047b8"}, - {file = "scipy-1.15.3-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6487aa99c2a3d509a5227d9a5e889ff05830a06b2ce08ec30df6d79db5fcd5c5"}, - {file = "scipy-1.15.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:50f9e62461c95d933d5c5ef4a1f2ebf9a2b4e83b0db374cb3f1de104d935922e"}, - {file = "scipy-1.15.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:14ed70039d182f411ffc74789a16df3835e05dc469b898233a245cdfd7f162cb"}, - {file = "scipy-1.15.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a769105537aa07a69468a0eefcd121be52006db61cdd8cac8a0e68980bbb723"}, - {file = "scipy-1.15.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9db984639887e3dffb3928d118145ffe40eff2fa40cb241a306ec57c219ebbbb"}, - {file = "scipy-1.15.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:40e54d5c7e7ebf1aa596c374c49fa3135f04648a0caabcb66c52884b943f02b4"}, - {file = "scipy-1.15.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5e721fed53187e71d0ccf382b6bf977644c533e506c4d33c3fb24de89f5c3ed5"}, - {file = "scipy-1.15.3-cp313-cp313t-win_amd64.whl", hash = "sha256:76ad1fb5f8752eabf0fa02e4cc0336b4e8f021e2d5f061ed37d6d264db35e3ca"}, - {file = "scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf"}, -] - -[package.dependencies] -numpy = ">=1.23.5,<2.5" - -[package.extras] -dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] -doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.19.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.0.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)"] -test = ["Cython", "array-api-strict (>=2.0,<2.1.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja ; sys_platform != \"emscripten\"", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] - [[package]] name = "scipy" version = "1.17.1" @@ -8617,7 +6690,6 @@ description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = ">=3.11" groups = ["main"] -markers = "python_version >= \"3.11\"" files = [ {file = "scipy-1.17.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec"}, {file = "scipy-1.17.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696"}, @@ -8697,7 +6769,7 @@ description = "Unsupervised text tokenizer and detokenizer." optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "sentencepiece-0.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e10fa50bdbaa5e2445dbd387979980d391760faf0ec99a09bd7780ff37eaec44"}, {file = "sentencepiece-0.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f27ae6deea72efdb6f361750c92f6c21fd0ad087445082770cc34015213c526"}, @@ -8777,7 +6849,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "sentry_sdk-2.53.0-py2.py3-none-any.whl", hash = "sha256:46e1ed8d84355ae54406c924f6b290c3d61f4048625989a723fd622aab838899"}, {file = "sentry_sdk-2.53.0.tar.gz", hash = "sha256:6520ef2c4acd823f28efc55e43eb6ce2e6d9f954a95a3aa96b6fd14871e92b77"}, @@ -8841,7 +6913,7 @@ description = "A Python module to customize the process title" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "setproctitle-1.3.7-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cf555b6299f10a6eb44e4f96d2f5a3884c70ce25dc5c8796aaa2f7b40e72cb1b"}, {file = "setproctitle-1.3.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:690b4776f9c15aaf1023bb07d7c5b797681a17af98a4a69e76a1d504e41108b7"}, @@ -8952,7 +7024,7 @@ description = "Easily download, build, install, upgrade, and uninstall Python pa optional = false python-versions = ">=3.9" groups = ["main"] -markers = "python_full_version == \"3.13.0\" or python_version == \"3.12\"" +markers = "python_version == \"3.12\"" files = [ {file = "setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173"}, {file = "setuptools-80.10.2.tar.gz", hash = "sha256:8b0e9d10c784bf7d262c4e5ec5d4ec94127ce206e8738f29a437945fbc219b70"}, @@ -8974,6 +7046,7 @@ description = "Easily download, build, install, upgrade, and uninstall Python pa optional = false python-versions = ">=3.9" groups = ["main"] +markers = "python_version == \"3.11\"" files = [ {file = "setuptools-82.0.0-py3-none-any.whl", hash = "sha256:70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0"}, {file = "setuptools-82.0.0.tar.gz", hash = "sha256:22e0a2d69474c6ae4feb01951cb69d515ed23728cf96d05513d36e42b62b37cb"}, @@ -8992,9 +7065,10 @@ type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.deve name = "shellingham" version = "1.5.4" description = "Tool to Detect Surrounding Shell" -optional = false +optional = true python-versions = ">=3.7" groups = ["main"] +markers = "extra == \"vllm\"" files = [ {file = "shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686"}, {file = "shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de"}, @@ -9139,7 +7213,7 @@ description = "Utils for streaming large files (S3, HDFS, GCS, SFTP, Azure Blob optional = true python-versions = "<4.0,>=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "smart_open-7.5.1-py3-none-any.whl", hash = "sha256:3e07cbbd9c8a908bcb8e25d48becf1a5cbb4886fa975e9f34c672ed171df2318"}, {file = "smart_open-7.5.1.tar.gz", hash = "sha256:3f08e16827c4733699e6b2cc40328a3568f900cb12ad9a3ad233ba6c872d9fe7"}, @@ -9166,7 +7240,7 @@ description = "Sniff out which async library your code is running under" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"client\"" +markers = "extra == \"vllm\" or extra == \"tinker\"" files = [ {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, @@ -9475,7 +7549,7 @@ description = "SSE plugin for Starlette" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "sse_starlette-3.3.2-py3-none-any.whl", hash = "sha256:5c3ea3dad425c601236726af2f27689b74494643f57017cafcb6f8c9acfbb862"}, {file = "sse_starlette-3.3.2.tar.gz", hash = "sha256:678fca55a1945c734d8472a6cad186a55ab02840b4f6786f5ee8770970579dcd"}, @@ -9517,7 +7591,7 @@ description = "A system for controlling process state under UNIX" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "supervisor-4.3.0-py2.py3-none-any.whl", hash = "sha256:0bcb763fddafba410f35cbde226aa7f8514b9fb82eb05a0c85f6588d1c13f8db"}, {file = "supervisor-4.3.0.tar.gz", hash = "sha256:4a2bf149adf42997e1bb44b70c43b613275ec9852c3edacca86a9166b27e945e"}, @@ -9526,44 +7600,6 @@ files = [ [package.extras] test = ["pytest", "pytest-cov"] -[[package]] -name = "swanlab" -version = "0.10.0" -description = "Python library for streamlined tracking and management of AI training processes." -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "swanlab-0.10.0-py3-none-any.whl", hash = "sha256:5467143d68925ee0968e3409174fbf25958a7736b0c6f347c90aabaf257f9057"}, - {file = "swanlab-0.10.0.tar.gz", hash = "sha256:746ca2798a4ca40a5d0cb2535993b0f4c88348b7a182da813a4b45feda24eef9"}, -] - -[package.dependencies] -click = "*" -nvidia-ml-py = "*" -orjson = {version = "*", markers = "python_version > \"3.9\""} -platformdirs = ">=4.2.0" -protobuf = {version = ">=6.31.1,<7", markers = "python_version > \"3.9\" or sys_platform != \"linux\""} -psutil = ">=5.0.0" -pwinput = ">=1.0.3" -pydantic = ">=2.0,<3.0" -pydantic-settings = ">=2.8.1" -pyecharts = ">=2.0.0" -pyroaring = {version = ">=1.0.0,<1.0.4", markers = "python_version < \"3.14\""} -pyyaml = "*" -requests = ">=2.28.0" -rich = ">=13.6.0" -setuptools = "*" -urllib3 = ">=1.26.0" -watchdog = "*" -wrapt = ">=1.17.0" - -[package.extras] -dashboard = ["swanboard (==0.1.10b2)"] -media = ["imageio[ffmpeg]", "matplotlib", "moviepy", "numpy", "pillow", "rdkit (<=2023.9.6) ; python_version == \"3.9\"", "rdkit (>=2025.9.6) ; python_version > \"3.9\"", "soundfile"] -s3 = ["boto3 (>=1.35.49)"] - [[package]] name = "sympy" version = "1.14.0" @@ -9589,7 +7625,7 @@ description = "Pretty-print tabular data" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f"}, {file = "tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c"}, @@ -9598,85 +7634,6 @@ files = [ [package.extras] widechars = ["wcwidth"] -[[package]] -name = "tensordict" -version = "0.14.2" -description = "TensorDict is a pytorch dedicated tensor container." -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "tensordict-0.14.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:9e4650b76d4e0672cf9c0320225b21434a589f4629d5836114019335875ffee9"}, - {file = "tensordict-0.14.2-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b9963fdd3bdcfea7919eea15208d5730c7862bcb028efd26fb46b6072a2abc79"}, - {file = "tensordict-0.14.2-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:404f47a5d024e1215792a920d7a742402981e907bc8a0c86f374358e719fefb3"}, - {file = "tensordict-0.14.2-cp310-cp310-win_amd64.whl", hash = "sha256:45c851e2b884c2e63180d75a82599ccb343b816a4ab3da4b28290695726184df"}, - {file = "tensordict-0.14.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:e2ac08238d0404a7f17f938331918c9bea026123bc7df4c07f2ce52b3068953f"}, - {file = "tensordict-0.14.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:001c62b1b9b7305077f29ab4e1a06693f1a7add7bfb2cd42d03b2172f185d425"}, - {file = "tensordict-0.14.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:002ce1387d1b6bb2ee29affbb68aaae4c79fa8773137bef88534fc2fadefca2f"}, - {file = "tensordict-0.14.2-cp311-cp311-win_amd64.whl", hash = "sha256:0d1f8f949db128b896b9cb022de532d3d87902811d2751c11f52270a27eabd9a"}, - {file = "tensordict-0.14.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:0e8b65def0da306741d262976e7613afb2079e8b521b6be910edd8080a33c265"}, - {file = "tensordict-0.14.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1553be455faf595bdc0e2b18649ca8791728ea13359805c22fe8c2aff6bd8fd0"}, - {file = "tensordict-0.14.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0a748755bfb35db40123bcda67bfebac0138283d25deda76fe30be58ce10c247"}, - {file = "tensordict-0.14.2-cp312-cp312-win_amd64.whl", hash = "sha256:5e676f6048c4e3ad53de379469270545526d6a19badbc48bd5870b68c64e84f4"}, - {file = "tensordict-0.14.2-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:8c5e4aa2a4a8b4b1370828186836fad9605f2091a3fa3e009b15423d787aaed3"}, - {file = "tensordict-0.14.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7d79bf7bcbdf6b0686ba756debffe9b73de9686e9fa5923747538408f130df9d"}, - {file = "tensordict-0.14.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:75a2a55f4633fbd3b82d158b2442e561bd595d48138ca473a2008e4e8689e640"}, - {file = "tensordict-0.14.2-cp313-cp313-win_amd64.whl", hash = "sha256:99a36acdc59eb162d3de214be464be2c85998f004b83ddb425aa23f74b92b6f6"}, - {file = "tensordict-0.14.2-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:10ae025f99e23dcec102652a9859c609122abac053fc3cd09ad999959d9069f2"}, - {file = "tensordict-0.14.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:4ec2d0d1ca4a61196deadb7a88584d9df47dc720ba45d00af15783396961295f"}, - {file = "tensordict-0.14.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:6d710cbc3091be3fee8d009492fb372df4c732e064495cf0ed5cdfacb5e168bd"}, - {file = "tensordict-0.14.2-cp314-cp314-win_amd64.whl", hash = "sha256:31512077ade1931b47b90be02a96ebfea23db28ab195c158bea01e11a0c71638"}, - {file = "tensordict-0.14.2-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:ad9c5a260b031145cd8a10a2fcdc5aa6c817c0e92cb80ec40bb140a0368b868a"}, - {file = "tensordict-0.14.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:595cdb56991ea6a7880af363f349299bb432041aa117fd8cdde76f0976f335a3"}, - {file = "tensordict-0.14.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:0482dd97ba526d4918f8920aaa3936f6fe32f2ed4682a714e779582eacba4c2c"}, - {file = "tensordict-0.14.2-cp314-cp314t-win_amd64.whl", hash = "sha256:68cba296f45c4eb98587644ce07902fbe75e064a2b738b8c5b2b590b8ea8f85b"}, -] - -[package.dependencies] -cloudpickle = "*" -importlib_metadata = "*" -numpy = "*" -orjson = {version = "*", markers = "python_version < \"3.13\""} -packaging = "*" -pyvers = ">=0.2.0,<0.3.0" -torch = "*" - -[package.extras] -dev = ["ninja", "pybind11 (>=2.13)"] -h5 = ["h5py (>=3.8)"] -onnx = ["onnx", "onnxruntime", "onnxscript"] -pandas = ["pandas (>=1.5)"] -parquet = ["pyarrow (>=10.0)"] -tabular = ["pandas (>=1.5)", "pyarrow (>=10.0)"] -tests = ["pytest", "pytest-benchmark", "pytest-instafail", "pytest-rerunfailures", "pyyaml"] -typecheck = ["mypy (>=1.0.0)"] -zarr = ["zarr (>=3.0) ; python_version >= \"3.11\""] - -[[package]] -name = "textual" -version = "8.2.8" -description = "Modern Text User Interface framework" -optional = true -python-versions = "<4.0,>=3.9" -groups = ["main"] -markers = "extra == \"client\"" -files = [ - {file = "textual-8.2.8-py3-none-any.whl", hash = "sha256:267375fd402dc8d981457212efa71f0e3365fd17bba144ba9bb3ed7563cb374a"}, - {file = "textual-8.2.8.tar.gz", hash = "sha256:3f106a9fbc73e39dd266c9712432087de78a6d644084c7c241d6a25c3169115b"}, -] - -[package.dependencies] -markdown-it-py = {version = ">=2.1.0", extras = ["linkify"]} -mdit-py-plugins = "*" -platformdirs = ">=3.6.0,<5" -pygments = ">=2.19.2,<3.0.0" -rich = ">=14.2.0" -typing-extensions = ">=4.4.0,<5.0.0" - -[package.extras] -syntax = ["tree-sitter (>=0.25.0) ; python_version >= \"3.10\"", "tree-sitter-bash (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-css (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-go (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-html (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-java (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-javascript (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-json (>=0.24.0) ; python_version >= \"3.10\"", "tree-sitter-markdown (>=0.3.0) ; python_version >= \"3.10\"", "tree-sitter-python (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-regex (>=0.24.0) ; python_version >= \"3.10\"", "tree-sitter-rust (>=0.23.0) ; python_version >= \"3.10\"", "tree-sitter-sql (>=0.3.11) ; python_version >= \"3.10\"", "tree-sitter-toml (>=0.6.0) ; python_version >= \"3.10\"", "tree-sitter-xml (>=0.7.0) ; python_version >= \"3.10\"", "tree-sitter-yaml (>=0.6.0) ; python_version >= \"3.10\""] - [[package]] name = "tiktoken" version = "0.12.0" @@ -9684,7 +7641,7 @@ description = "tiktoken is a fast BPE tokeniser for use with OpenAI's models" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"eval\" or extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "tiktoken-0.12.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3de02f5a491cfd179aec916eddb70331814bd6bf764075d39e21d5862e533970"}, {file = "tiktoken-0.12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b6cfb6d9b7b54d20af21a912bfe63a2727d9cfa8fbda642fd8322c70340aad16"}, @@ -9754,15 +7711,15 @@ blobfile = ["blobfile (>=2)"] [[package]] name = "tinker" -version = "0.29.0" +version = "0.14.0" description = "The official Python SDK for the tinker API" optional = true python-versions = ">=3.11" groups = ["main"] -markers = "python_version >= \"3.11\" and extra == \"client\"" +markers = "extra == \"tinker\"" files = [ - {file = "tinker-0.29.0-py3-none-any.whl", hash = "sha256:d3a9edd08718edaa8d8f72fe4ffa2a7f1ae8da2967f35e1ab206988b86a1b1fb"}, - {file = "tinker-0.29.0.tar.gz", hash = "sha256:615ed096ff647addafd1637096e8c688ce1581b8132bb937426492d2c5c0d619"}, + {file = "tinker-0.14.0-py3-none-any.whl", hash = "sha256:b8e4dd2c77858e80177cb0b970c5d64a7dae085b67a711cc7e1d95352534c563"}, + {file = "tinker-0.14.0.tar.gz", hash = "sha256:554b30c43d9cf83b41eb07775a2a9f65eb2e77bed2cd79792bd8cd0d7693e717"}, ] [package.dependencies] @@ -9771,15 +7728,11 @@ click = ">=8.0.0" distro = ">=1.7.0,<2" httpx = {version = ">=0.23.0,<1", extras = ["http2"]} numpy = "*" -orjson = ">=3.10.0" -protobuf = ">=4.21" pydantic = ">=1.9.0,<3" -pyqwest = ">=0.4.1" rich = ">=13.0.0" sniffio = "*" transformers = "*" typing-extensions = ">=4.10,<5" -zstandard = ">=0.24.0" [package.extras] aiohttp = ["aiohttp", "httpx-aiohttp (>=0.1.8)"] @@ -9827,64 +7780,6 @@ dev = ["tokenizers[testing]"] docs = ["setuptools-rust", "sphinx", "sphinx-rtd-theme"] testing = ["datasets", "numpy", "pytest", "pytest-asyncio", "requests", "ruff", "ty"] -[[package]] -name = "tomli" -version = "2.4.1" -description = "A lil' TOML parser" -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.10\" and (extra == \"rl\" or extra == \"async-rl\" or extra == \"test\")" -files = [ - {file = "tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30"}, - {file = "tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a"}, - {file = "tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076"}, - {file = "tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9"}, - {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c"}, - {file = "tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc"}, - {file = "tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049"}, - {file = "tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e"}, - {file = "tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece"}, - {file = "tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a"}, - {file = "tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085"}, - {file = "tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9"}, - {file = "tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5"}, - {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585"}, - {file = "tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1"}, - {file = "tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917"}, - {file = "tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9"}, - {file = "tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257"}, - {file = "tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54"}, - {file = "tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a"}, - {file = "tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897"}, - {file = "tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f"}, - {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d"}, - {file = "tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5"}, - {file = "tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd"}, - {file = "tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36"}, - {file = "tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd"}, - {file = "tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf"}, - {file = "tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac"}, - {file = "tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662"}, - {file = "tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853"}, - {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15"}, - {file = "tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba"}, - {file = "tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6"}, - {file = "tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7"}, - {file = "tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232"}, - {file = "tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4"}, - {file = "tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c"}, - {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d"}, - {file = "tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41"}, - {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c"}, - {file = "tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f"}, - {file = "tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8"}, - {file = "tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26"}, - {file = "tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396"}, - {file = "tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe"}, - {file = "tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f"}, -] - [[package]] name = "torch" version = "2.9.1" @@ -9960,7 +7855,7 @@ description = "An audio package for PyTorch" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "torchaudio-2.9.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd13541197e035338bd43225b2067532056486d357c661e12d49ace4fc37f8bb"}, {file = "torchaudio-2.9.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:31ec46b718b7caa0182221bfb42e2ad223947b752a996dcdc0388c34a678c966"}, @@ -10002,7 +7897,7 @@ description = "image and video datasets and models for torch deep learning" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"transformers\" or extra == \"vllm\"" files = [ {file = "torchvision-0.24.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ded5e625788572e4e1c4d155d1bbc48805c113794100d70e19c76e39e4d53465"}, {file = "torchvision-0.24.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:54ed17c3d30e718e08d8da3fd5b30ea44b0311317e55647cb97077a29ecbc25b"}, @@ -10065,35 +7960,6 @@ notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] -[[package]] -name = "transferqueue" -version = "0.1.10" -description = "TransferQueue: An Asynchronous Streaming Data Management Module" -optional = true -python-versions = ">=3.10" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "transferqueue-0.1.10-py3-none-any.whl", hash = "sha256:a8efceb971fe57d52ac7ddd86d9a6ef63f1824819dddc2ea980a96b73ff2f8ea"}, -] - -[package.dependencies] -hydra-core = "*" -msgspec = "*" -numpy = "*" -omegaconf = "*" -prometheus_client = ">=0.20.0" -psutil = "*" -pyzmq = "*" -ray = {version = "*", extras = ["default"]} -tensordict = ">=0.10.0" - -[package.extras] -build = ["build"] -mooncake = ["cuda-python", "mooncake-transfer-engine (>=0.3.10.post2)"] -test = ["flake8", "pytest (>=7.0.0)", "pytest-asyncio (>=0.20.0)", "pytest-mock"] -yuanrong = ["openyuanrong-datasystem"] - [[package]] name = "transformer-engine" version = "2.12.0" @@ -10268,9 +8134,10 @@ tutorials = ["matplotlib", "pandas", "tabulate"] name = "typer" version = "0.24.1" description = "Typer, build great CLIs. Easy to code. Based on Python type hints." -optional = false +optional = true python-versions = ">=3.10" groups = ["main"] +markers = "extra == \"vllm\"" files = [ {file = "typer-0.24.1-py3-none-any.whl", hash = "sha256:112c1f0ce578bfb4cab9ffdabc68f031416ebcc216536611ba21f04e9aa84c9e"}, {file = "typer-0.24.1.tar.gz", hash = "sha256:e39b4732d65fbdcde189ae76cf7cd48aeae72919dea1fdfc16593be016256b45"}, @@ -10316,7 +8183,7 @@ description = "Provider of IANA time zone data" optional = false python-versions = ">=2" groups = ["main"] -markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\" or python_version == \"3.10\"" +markers = "sys_platform == \"win32\" or sys_platform == \"emscripten\"" files = [ {file = "tzdata-2025.3-py2.py3-none-any.whl", hash = "sha256:06a47e5700f3081aab02b2e513160914ff0694bce9947d6b76ebd6bf57cfc5d1"}, {file = "tzdata-2025.3.tar.gz", hash = "sha256:de39c2ca5dc7b0344f2eba86f49d614019d29f060fc4ebc8a417896a620b56a7"}, @@ -10347,7 +8214,7 @@ description = "The lightning-fast ASGI server." optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "uvicorn-0.41.0-py3-none-any.whl", hash = "sha256:29e35b1d2c36a04b9e180d4007ede3bcb32a85fbdfd6c6aeb3f26839de088187"}, {file = "uvicorn-0.41.0.tar.gz", hash = "sha256:09d11cf7008da33113824ee5a1c6422d89fbc2ff476540d69a34c87fab8b571a"}, @@ -10360,7 +8227,6 @@ h11 = ">=0.8" httptools = {version = ">=0.6.3", optional = true, markers = "extra == \"standard\""} python-dotenv = {version = ">=0.13", optional = true, markers = "extra == \"standard\""} pyyaml = {version = ">=5.1", optional = true, markers = "extra == \"standard\""} -typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} uvloop = {version = ">=0.15.1", optional = true, markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and extra == \"standard\""} watchfiles = {version = ">=0.20", optional = true, markers = "extra == \"standard\""} websockets = {version = ">=10.4", optional = true, markers = "extra == \"standard\""} @@ -10375,7 +8241,7 @@ description = "Fast implementation of asyncio event loop on top of libuv" optional = true python-versions = ">=3.8.1" groups = ["main"] -markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and (extra == \"rl\" or extra == \"async-rl\")" +markers = "sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\" and (extra == \"vllm\" or extra == \"ray\")" files = [ {file = "uvloop-0.22.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:ef6f0d4cc8a9fa1f6a910230cd53545d9a14479311e87e3cb225495952eb672c"}, {file = "uvloop-0.22.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7cd375a12b71d33d46af85a3343b35d98e8116134ba404bd657b3b1d15988792"}, @@ -10440,7 +8306,7 @@ description = "Virtual Python Environment builder" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"ray\"" files = [ {file = "virtualenv-21.1.0-py3-none-any.whl", hash = "sha256:164f5e14c5587d170cf98e60378eb91ea35bf037be313811905d3a24ea33cc07"}, {file = "virtualenv-21.1.0.tar.gz", hash = "sha256:1990a0188c8f16b6b9cf65c9183049007375b26aad415514d377ccacf1e4fb44"}, @@ -10451,7 +8317,6 @@ distlib = ">=0.3.7,<1" filelock = {version = ">=3.24.2,<4", markers = "python_version >= \"3.10\""} platformdirs = ">=3.9.1,<5" python-discovery = ">=1" -typing-extensions = {version = ">=4.13.2", markers = "python_version < \"3.11\""} [[package]] name = "vllm" @@ -10460,7 +8325,7 @@ description = "A high-throughput and memory-efficient inference and serving engi optional = true python-versions = "<3.14,>=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\"" files = [ {file = "vllm-0.16.0-cp38-abi3-manylinux_2_31_aarch64.whl", hash = "sha256:dfaa14846608fd229dda9d372e2ad3f13854fd09147c2ba36b40579cf3c03804"}, {file = "vllm-0.16.0-cp38-abi3-manylinux_2_31_x86_64.whl", hash = "sha256:f066b2a2f8597a4a3ada8fbbfd122b59086864b2260ca42dc81bf9fb57af0c42"}, @@ -10539,50 +8404,6 @@ petit-kernel = ["petit-kernel"] runai = ["runai-model-streamer[gcs,s3] (>=0.15.3)"] tensorizer = ["tensorizer (==2.10.1)"] -[[package]] -name = "watchdog" -version = "6.0.0" -description = "Filesystem events monitoring" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26"}, - {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112"}, - {file = "watchdog-6.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c897ac1b55c5a1461e16dae288d22bb2e412ba9807df8397a635d88f671d36c3"}, - {file = "watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c"}, - {file = "watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2"}, - {file = "watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c"}, - {file = "watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948"}, - {file = "watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860"}, - {file = "watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0"}, - {file = "watchdog-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:490ab2ef84f11129844c23fb14ecf30ef3d8a6abafd3754a6f75ca1e6654136c"}, - {file = "watchdog-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:76aae96b00ae814b181bb25b1b98076d5fc84e8a53cd8885a318b42b6d3a5134"}, - {file = "watchdog-6.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a175f755fc2279e0b7312c0035d52e27211a5bc39719dd529625b1930917345b"}, - {file = "watchdog-6.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e6f0e77c9417e7cd62af82529b10563db3423625c5fce018430b249bf977f9e8"}, - {file = "watchdog-6.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:90c8e78f3b94014f7aaae121e6b909674df5b46ec24d6bebc45c44c56729af2a"}, - {file = "watchdog-6.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e7631a77ffb1f7d2eefa4445ebbee491c720a5661ddf6df3498ebecae5ed375c"}, - {file = "watchdog-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c7ac31a19f4545dd92fc25d200694098f42c9a8e391bc00bdd362c5736dbf881"}, - {file = "watchdog-6.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9513f27a1a582d9808cf21a07dae516f0fab1cf2d7683a742c498b93eedabb11"}, - {file = "watchdog-6.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7a0e56874cfbc4b9b05c60c8a1926fedf56324bb08cfbc188969777940aef3aa"}, - {file = "watchdog-6.0.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:e6439e374fc012255b4ec786ae3c4bc838cd7309a540e5fe0952d03687d8804e"}, - {file = "watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13"}, - {file = "watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379"}, - {file = "watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e"}, - {file = "watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f"}, - {file = "watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26"}, - {file = "watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c"}, - {file = "watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2"}, - {file = "watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a"}, - {file = "watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680"}, - {file = "watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f"}, - {file = "watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282"}, -] - -[package.extras] -watchmedo = ["PyYAML (>=3.10)"] - [[package]] name = "watchfiles" version = "1.1.1" @@ -10590,7 +8411,7 @@ description = "Simple, modern and high performance file watching and code reload optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "watchfiles-1.1.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:eef58232d32daf2ac67f42dea51a2c80f0d03379075d44a587051e63cc2e368c"}, {file = "watchfiles-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:03fa0f5237118a0c5e496185cafa92878568b652a2e9a9382a5151b1a0380a43"}, @@ -10706,19 +8527,6 @@ files = [ [package.dependencies] anyio = ">=3.0.0" -[[package]] -name = "wcwidth" -version = "0.8.3" -description = "Measures the displayed width of unicode strings in a terminal" -optional = true -python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"async-rl\"" -files = [ - {file = "wcwidth-0.8.3-py3-none-any.whl", hash = "sha256:d5b73dba6158a595ec9370350e7f2637bcac8d6c5e4fde34f30fcffb6103a5e4"}, - {file = "wcwidth-0.8.3.tar.gz", hash = "sha256:d128512515fbf4612e0ff21fd6380399210318b7b54a9af59dff8454cf9730eb"}, -] - [[package]] name = "websockets" version = "16.0" @@ -10726,7 +8534,7 @@ description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" optional = true python-versions = ">=3.10" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\"" +markers = "extra == \"vllm\" or extra == \"ray\"" files = [ {file = "websockets-16.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:04cdd5d2d1dacbad0a7bf36ccbcd3ccd5a30ee188f2560b7a62a30d14107b31a"}, {file = "websockets-16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8ff32bb86522a9e5e31439a58addbb0166f0204d64066fb955265c4e214160f0"}, @@ -10798,7 +8606,7 @@ description = "A small Python utility to set file creation time on Windows" optional = true python-versions = ">=3.5" groups = ["main"] -markers = "sys_platform == \"win32\" and (extra == \"rl\" or extra == \"async-rl\")" +markers = "extra == \"vllm\" and sys_platform == \"win32\"" files = [ {file = "win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390"}, {file = "win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0"}, @@ -10807,109 +8615,93 @@ files = [ [package.extras] dev = ["black (>=19.3b0) ; python_version >= \"3.6\"", "pytest (>=4.6.2)"] -[[package]] -name = "word2number" -version = "1.1" -description = "Convert number words eg. three hundred and forty two to numbers (342)." -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "word2number-1.1.zip", hash = "sha256:70e27a5d387f67b04c71fbb7621c05930b19bfd26efd6851e6e0f9969dcde7d0"}, -] - [[package]] name = "wrapt" -version = "1.17.3" +version = "2.1.1" description = "Module for decorators, wrappers and monkey patching." optional = true -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"rl\" or extra == \"async-rl\" or extra == \"server\"" -files = [ - {file = "wrapt-1.17.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:88bbae4d40d5a46142e70d58bf664a89b6b4befaea7b2ecc14e03cedb8e06c04"}, - {file = "wrapt-1.17.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e6b13af258d6a9ad602d57d889f83b9d5543acd471eee12eb51f5b01f8eb1bc2"}, - {file = "wrapt-1.17.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd341868a4b6714a5962c1af0bd44f7c404ef78720c7de4892901e540417111c"}, - {file = "wrapt-1.17.3-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f9b2601381be482f70e5d1051a5965c25fb3625455a2bf520b5a077b22afb775"}, - {file = "wrapt-1.17.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:343e44b2a8e60e06a7e0d29c1671a0d9951f59174f3709962b5143f60a2a98bd"}, - {file = "wrapt-1.17.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:33486899acd2d7d3066156b03465b949da3fd41a5da6e394ec49d271baefcf05"}, - {file = "wrapt-1.17.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:e6f40a8aa5a92f150bdb3e1c44b7e98fb7113955b2e5394122fa5532fec4b418"}, - {file = "wrapt-1.17.3-cp310-cp310-win32.whl", hash = "sha256:a36692b8491d30a8c75f1dfee65bef119d6f39ea84ee04d9f9311f83c5ad9390"}, - {file = "wrapt-1.17.3-cp310-cp310-win_amd64.whl", hash = "sha256:afd964fd43b10c12213574db492cb8f73b2f0826c8df07a68288f8f19af2ebe6"}, - {file = "wrapt-1.17.3-cp310-cp310-win_arm64.whl", hash = "sha256:af338aa93554be859173c39c85243970dc6a289fa907402289eeae7543e1ae18"}, - {file = "wrapt-1.17.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:273a736c4645e63ac582c60a56b0acb529ef07f78e08dc6bfadf6a46b19c0da7"}, - {file = "wrapt-1.17.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5531d911795e3f935a9c23eb1c8c03c211661a5060aab167065896bbf62a5f85"}, - {file = "wrapt-1.17.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0610b46293c59a3adbae3dee552b648b984176f8562ee0dba099a56cfbe4df1f"}, - {file = "wrapt-1.17.3-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b32888aad8b6e68f83a8fdccbf3165f5469702a7544472bdf41f582970ed3311"}, - {file = "wrapt-1.17.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cccf4f81371f257440c88faed6b74f1053eef90807b77e31ca057b2db74edb1"}, - {file = "wrapt-1.17.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8a210b158a34164de8bb68b0e7780041a903d7b00c87e906fb69928bf7890d5"}, - {file = "wrapt-1.17.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:79573c24a46ce11aab457b472efd8d125e5a51da2d1d24387666cd85f54c05b2"}, - {file = "wrapt-1.17.3-cp311-cp311-win32.whl", hash = "sha256:c31eebe420a9a5d2887b13000b043ff6ca27c452a9a22fa71f35f118e8d4bf89"}, - {file = "wrapt-1.17.3-cp311-cp311-win_amd64.whl", hash = "sha256:0b1831115c97f0663cb77aa27d381237e73ad4f721391a9bfb2fe8bc25fa6e77"}, - {file = "wrapt-1.17.3-cp311-cp311-win_arm64.whl", hash = "sha256:5a7b3c1ee8265eb4c8f1b7d29943f195c00673f5ab60c192eba2d4a7eae5f46a"}, - {file = "wrapt-1.17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ab232e7fdb44cdfbf55fc3afa31bcdb0d8980b9b95c38b6405df2acb672af0e0"}, - {file = "wrapt-1.17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:9baa544e6acc91130e926e8c802a17f3b16fbea0fd441b5a60f5cf2cc5c3deba"}, - {file = "wrapt-1.17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6b538e31eca1a7ea4605e44f81a48aa24c4632a277431a6ed3f328835901f4fd"}, - {file = "wrapt-1.17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:042ec3bb8f319c147b1301f2393bc19dba6e176b7da446853406d041c36c7828"}, - {file = "wrapt-1.17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3af60380ba0b7b5aeb329bc4e402acd25bd877e98b3727b0135cb5c2efdaefe9"}, - {file = "wrapt-1.17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0b02e424deef65c9f7326d8c19220a2c9040c51dc165cddb732f16198c168396"}, - {file = "wrapt-1.17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:74afa28374a3c3a11b3b5e5fca0ae03bef8450d6aa3ab3a1e2c30e3a75d023dc"}, - {file = "wrapt-1.17.3-cp312-cp312-win32.whl", hash = "sha256:4da9f45279fff3543c371d5ababc57a0384f70be244de7759c85a7f989cb4ebe"}, - {file = "wrapt-1.17.3-cp312-cp312-win_amd64.whl", hash = "sha256:e71d5c6ebac14875668a1e90baf2ea0ef5b7ac7918355850c0908ae82bcb297c"}, - {file = "wrapt-1.17.3-cp312-cp312-win_arm64.whl", hash = "sha256:604d076c55e2fdd4c1c03d06dc1a31b95130010517b5019db15365ec4a405fc6"}, - {file = "wrapt-1.17.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a47681378a0439215912ef542c45a783484d4dd82bac412b71e59cf9c0e1cea0"}, - {file = "wrapt-1.17.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:54a30837587c6ee3cd1a4d1c2ec5d24e77984d44e2f34547e2323ddb4e22eb77"}, - {file = "wrapt-1.17.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:16ecf15d6af39246fe33e507105d67e4b81d8f8d2c6598ff7e3ca1b8a37213f7"}, - {file = "wrapt-1.17.3-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6fd1ad24dc235e4ab88cda009e19bf347aabb975e44fd5c2fb22a3f6e4141277"}, - {file = "wrapt-1.17.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ed61b7c2d49cee3c027372df5809a59d60cf1b6c2f81ee980a091f3afed6a2d"}, - {file = "wrapt-1.17.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:423ed5420ad5f5529db9ce89eac09c8a2f97da18eb1c870237e84c5a5c2d60aa"}, - {file = "wrapt-1.17.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e01375f275f010fcbf7f643b4279896d04e571889b8a5b3f848423d91bf07050"}, - {file = "wrapt-1.17.3-cp313-cp313-win32.whl", hash = "sha256:53e5e39ff71b3fc484df8a522c933ea2b7cdd0d5d15ae82e5b23fde87d44cbd8"}, - {file = "wrapt-1.17.3-cp313-cp313-win_amd64.whl", hash = "sha256:1f0b2f40cf341ee8cc1a97d51ff50dddb9fcc73241b9143ec74b30fc4f44f6cb"}, - {file = "wrapt-1.17.3-cp313-cp313-win_arm64.whl", hash = "sha256:7425ac3c54430f5fc5e7b6f41d41e704db073309acfc09305816bc6a0b26bb16"}, - {file = "wrapt-1.17.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cf30f6e3c077c8e6a9a7809c94551203c8843e74ba0c960f4a98cd80d4665d39"}, - {file = "wrapt-1.17.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e228514a06843cae89621384cfe3a80418f3c04aadf8a3b14e46a7be704e4235"}, - {file = "wrapt-1.17.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5ea5eb3c0c071862997d6f3e02af1d055f381b1d25b286b9d6644b79db77657c"}, - {file = "wrapt-1.17.3-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:281262213373b6d5e4bb4353bc36d1ba4084e6d6b5d242863721ef2bf2c2930b"}, - {file = "wrapt-1.17.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc4a8d2b25efb6681ecacad42fca8859f88092d8732b170de6a5dddd80a1c8fa"}, - {file = "wrapt-1.17.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:373342dd05b1d07d752cecbec0c41817231f29f3a89aa8b8843f7b95992ed0c7"}, - {file = "wrapt-1.17.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d40770d7c0fd5cbed9d84b2c3f2e156431a12c9a37dc6284060fb4bec0b7ffd4"}, - {file = "wrapt-1.17.3-cp314-cp314-win32.whl", hash = "sha256:fbd3c8319de8e1dc79d346929cd71d523622da527cca14e0c1d257e31c2b8b10"}, - {file = "wrapt-1.17.3-cp314-cp314-win_amd64.whl", hash = "sha256:e1a4120ae5705f673727d3253de3ed0e016f7cd78dc463db1b31e2463e1f3cf6"}, - {file = "wrapt-1.17.3-cp314-cp314-win_arm64.whl", hash = "sha256:507553480670cab08a800b9463bdb881b2edeed77dc677b0a5915e6106e91a58"}, - {file = "wrapt-1.17.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:ed7c635ae45cfbc1a7371f708727bf74690daedc49b4dba310590ca0bd28aa8a"}, - {file = "wrapt-1.17.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:249f88ed15503f6492a71f01442abddd73856a0032ae860de6d75ca62eed8067"}, - {file = "wrapt-1.17.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a03a38adec8066d5a37bea22f2ba6bbf39fcdefbe2d91419ab864c3fb515454"}, - {file = "wrapt-1.17.3-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5d4478d72eb61c36e5b446e375bbc49ed002430d17cdec3cecb36993398e1a9e"}, - {file = "wrapt-1.17.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223db574bb38637e8230eb14b185565023ab624474df94d2af18f1cdb625216f"}, - {file = "wrapt-1.17.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e405adefb53a435f01efa7ccdec012c016b5a1d3f35459990afc39b6be4d5056"}, - {file = "wrapt-1.17.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:88547535b787a6c9ce4086917b6e1d291aa8ed914fdd3a838b3539dc95c12804"}, - {file = "wrapt-1.17.3-cp314-cp314t-win32.whl", hash = "sha256:41b1d2bc74c2cac6f9074df52b2efbef2b30bdfe5f40cb78f8ca22963bc62977"}, - {file = "wrapt-1.17.3-cp314-cp314t-win_amd64.whl", hash = "sha256:73d496de46cd2cdbdbcce4ae4bcdb4afb6a11234a1df9c085249d55166b95116"}, - {file = "wrapt-1.17.3-cp314-cp314t-win_arm64.whl", hash = "sha256:f38e60678850c42461d4202739f9bf1e3a737c7ad283638251e79cc49effb6b6"}, - {file = "wrapt-1.17.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:70d86fa5197b8947a2fa70260b48e400bf2ccacdcab97bb7de47e3d1e6312225"}, - {file = "wrapt-1.17.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:df7d30371a2accfe4013e90445f6388c570f103d61019b6b7c57e0265250072a"}, - {file = "wrapt-1.17.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:caea3e9c79d5f0d2c6d9ab96111601797ea5da8e6d0723f77eabb0d4068d2b2f"}, - {file = "wrapt-1.17.3-cp38-cp38-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:758895b01d546812d1f42204bd443b8c433c44d090248bf22689df673ccafe00"}, - {file = "wrapt-1.17.3-cp38-cp38-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:02b551d101f31694fc785e58e0720ef7d9a10c4e62c1c9358ce6f63f23e30a56"}, - {file = "wrapt-1.17.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:656873859b3b50eeebe6db8b1455e99d90c26ab058db8e427046dbc35c3140a5"}, - {file = "wrapt-1.17.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:a9a2203361a6e6404f80b99234fe7fb37d1fc73487b5a78dc1aa5b97201e0f22"}, - {file = "wrapt-1.17.3-cp38-cp38-win32.whl", hash = "sha256:55cbbc356c2842f39bcc553cf695932e8b30e30e797f961860afb308e6b1bb7c"}, - {file = "wrapt-1.17.3-cp38-cp38-win_amd64.whl", hash = "sha256:ad85e269fe54d506b240d2d7b9f5f2057c2aa9a2ea5b32c66f8902f768117ed2"}, - {file = "wrapt-1.17.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:30ce38e66630599e1193798285706903110d4f057aab3168a34b7fdc85569afc"}, - {file = "wrapt-1.17.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:65d1d00fbfb3ea5f20add88bbc0f815150dbbde3b026e6c24759466c8b5a9ef9"}, - {file = "wrapt-1.17.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a7c06742645f914f26c7f1fa47b8bc4c91d222f76ee20116c43d5ef0912bba2d"}, - {file = "wrapt-1.17.3-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7e18f01b0c3e4a07fe6dfdb00e29049ba17eadbc5e7609a2a3a4af83ab7d710a"}, - {file = "wrapt-1.17.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f5f51a6466667a5a356e6381d362d259125b57f059103dd9fdc8c0cf1d14139"}, - {file = "wrapt-1.17.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:59923aa12d0157f6b82d686c3fd8e1166fa8cdfb3e17b42ce3b6147ff81528df"}, - {file = "wrapt-1.17.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:46acc57b331e0b3bcb3e1ca3b421d65637915cfcd65eb783cb2f78a511193f9b"}, - {file = "wrapt-1.17.3-cp39-cp39-win32.whl", hash = "sha256:3e62d15d3cfa26e3d0788094de7b64efa75f3a53875cdbccdf78547aed547a81"}, - {file = "wrapt-1.17.3-cp39-cp39-win_amd64.whl", hash = "sha256:1f23fa283f51c890eda8e34e4937079114c74b4c81d2b2f1f1d94948f5cc3d7f"}, - {file = "wrapt-1.17.3-cp39-cp39-win_arm64.whl", hash = "sha256:24c2ed34dc222ed754247a2702b1e1e89fdbaa4016f324b4b8f1a802d4ffe87f"}, - {file = "wrapt-1.17.3-py3-none-any.whl", hash = "sha256:7171ae35d2c33d326ac19dd8facb1e82e5fd04ef8c6c0e394d7af55a55051c22"}, - {file = "wrapt-1.17.3.tar.gz", hash = "sha256:f66eb08feaa410fe4eebd17f2a2c8e2e46d3476e9f8c783daa8e09e0faa666d0"}, -] +markers = "extra == \"ray\"" +files = [ + {file = "wrapt-2.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7e927375e43fd5a985b27a8992327c22541b6dede1362fc79df337d26e23604f"}, + {file = "wrapt-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c99544b6a7d40ca22195563b6d8bc3986ee8bb82f272f31f0670fe9440c869"}, + {file = "wrapt-2.1.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b2be3fa5f4efaf16ee7c77d0556abca35f5a18ad4ac06f0ef3904c3399010ce9"}, + {file = "wrapt-2.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:67c90c1ae6489a6cb1a82058902caa8006706f7b4e8ff766f943e9d2c8e608d0"}, + {file = "wrapt-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:05c0db35ccffd7480143e62df1e829d101c7b86944ae3be7e4869a7efa621f53"}, + {file = "wrapt-2.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:0c2ec9f616755b2e1e0bf4d0961f59bb5c2e7a77407e7e2c38ef4f7d2fdde12c"}, + {file = "wrapt-2.1.1-cp310-cp310-win32.whl", hash = "sha256:203ba6b3f89e410e27dbd30ff7dccaf54dcf30fda0b22aa1b82d560c7f9fe9a1"}, + {file = "wrapt-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:6f9426d9cfc2f8732922fc96198052e55c09bb9db3ddaa4323a18e055807410e"}, + {file = "wrapt-2.1.1-cp310-cp310-win_arm64.whl", hash = "sha256:69c26f51b67076b40714cff81bdd5826c0b10c077fb6b0678393a6a2f952a5fc"}, + {file = "wrapt-2.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6c366434a7fb914c7a5de508ed735ef9c133367114e1a7cb91dfb5cd806a1549"}, + {file = "wrapt-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5d6a2068bd2e1e19e5a317c8c0b288267eec4e7347c36bc68a6e378a39f19ee7"}, + {file = "wrapt-2.1.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:891ab4713419217b2aed7dd106c9200f64e6a82226775a0d2ebd6bef2ebd1747"}, + {file = "wrapt-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c8ef36a0df38d2dc9d907f6617f89e113c5892e0a35f58f45f75901af0ce7d81"}, + {file = "wrapt-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:76e9af3ebd86f19973143d4d592cbf3e970cf3f66ddee30b16278c26ae34b8ab"}, + {file = "wrapt-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ff562067485ebdeaef2fa3fe9b1876bc4e7b73762e0a01406ad81e2076edcebf"}, + {file = "wrapt-2.1.1-cp311-cp311-win32.whl", hash = "sha256:9e60a30aa0909435ec4ea2a3c53e8e1b50ac9f640c0e9fe3f21fd248a22f06c5"}, + {file = "wrapt-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:7d79954f51fcf84e5ec4878ab4aea32610d70145c5bbc84b3370eabfb1e096c2"}, + {file = "wrapt-2.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:d3ffc6b0efe79e08fd947605fd598515aebefe45e50432dc3b5cd437df8b1ada"}, + {file = "wrapt-2.1.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ab8e3793b239db021a18782a5823fcdea63b9fe75d0e340957f5828ef55fcc02"}, + {file = "wrapt-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7c0300007836373d1c2df105b40777986accb738053a92fe09b615a7a4547e9f"}, + {file = "wrapt-2.1.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2b27c070fd1132ab23957bcd4ee3ba707a91e653a9268dc1afbd39b77b2799f7"}, + {file = "wrapt-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b0e36d845e8b6f50949b6b65fc6cd279f47a1944582ed4ec8258cd136d89a64"}, + {file = "wrapt-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4aeea04a9889370fcfb1ef828c4cc583f36a875061505cd6cd9ba24d8b43cc36"}, + {file = "wrapt-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d88b46bb0dce9f74b6817bc1758ff2125e1ca9e1377d62ea35b6896142ab6825"}, + {file = "wrapt-2.1.1-cp312-cp312-win32.whl", hash = "sha256:63decff76ca685b5c557082dfbea865f3f5f6d45766a89bff8dc61d336348833"}, + {file = "wrapt-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:b828235d26c1e35aca4107039802ae4b1411be0fe0367dd5b7e4d90e562fcbcd"}, + {file = "wrapt-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:75128507413a9f1bcbe2db88fd18fbdbf80f264b82fa33a6996cdeaf01c52352"}, + {file = "wrapt-2.1.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ce9646e17fa7c3e2e7a87e696c7de66512c2b4f789a8db95c613588985a2e139"}, + {file = "wrapt-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:428cfc801925454395aa468ba7ddb3ed63dc0d881df7b81626cdd433b4e2b11b"}, + {file = "wrapt-2.1.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5797f65e4d58065a49088c3b32af5410751cd485e83ba89e5a45e2aa8905af98"}, + {file = "wrapt-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a2db44a71202c5ae4bb5f27c6d3afbc5b23053f2e7e78aa29704541b5dad789"}, + {file = "wrapt-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8d5350c3590af09c1703dd60ec78a7370c0186e11eaafb9dda025a30eee6492d"}, + {file = "wrapt-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d9b076411bed964e752c01b49fd224cc385f3a96f520c797d38412d70d08359"}, + {file = "wrapt-2.1.1-cp313-cp313-win32.whl", hash = "sha256:0bb7207130ce6486727baa85373503bf3334cc28016f6928a0fa7e19d7ecdc06"}, + {file = "wrapt-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:cbfee35c711046b15147b0ae7db9b976f01c9520e6636d992cd9e69e5e2b03b1"}, + {file = "wrapt-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:7d2756061022aebbf57ba14af9c16e8044e055c22d38de7bf40d92b565ecd2b0"}, + {file = "wrapt-2.1.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4814a3e58bc6971e46baa910ecee69699110a2bf06c201e24277c65115a20c20"}, + {file = "wrapt-2.1.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:106c5123232ab9b9f4903692e1fa0bdc231510098f04c13c3081f8ad71c3d612"}, + {file = "wrapt-2.1.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1a40b83ff2535e6e56f190aff123821eea89a24c589f7af33413b9c19eb2c738"}, + {file = "wrapt-2.1.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:789cea26e740d71cf1882e3a42bb29052bc4ada15770c90072cb47bf73fb3dbf"}, + {file = "wrapt-2.1.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ba49c14222d5e5c0ee394495a8655e991dc06cbca5398153aefa5ac08cd6ccd7"}, + {file = "wrapt-2.1.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ac8cda531fe55be838a17c62c806824472bb962b3afa47ecbd59b27b78496f4e"}, + {file = "wrapt-2.1.1-cp313-cp313t-win32.whl", hash = "sha256:b8af75fe20d381dd5bcc9db2e86a86d7fcfbf615383a7147b85da97c1182225b"}, + {file = "wrapt-2.1.1-cp313-cp313t-win_amd64.whl", hash = "sha256:45c5631c9b6c792b78be2d7352129f776dd72c605be2c3a4e9be346be8376d83"}, + {file = "wrapt-2.1.1-cp313-cp313t-win_arm64.whl", hash = "sha256:da815b9263947ac98d088b6414ac83507809a1d385e4632d9489867228d6d81c"}, + {file = "wrapt-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9aa1765054245bb01a37f615503290d4e207e3fd59226e78341afb587e9c1236"}, + {file = "wrapt-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:feff14b63a6d86c1eee33a57f77573649f2550935981625be7ff3cb7342efe05"}, + {file = "wrapt-2.1.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:81fc5f22d5fcfdbabde96bb3f5379b9f4476d05c6d524d7259dc5dfb501d3281"}, + {file = "wrapt-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:951b228ecf66def855d22e006ab9a1fc12535111ae7db2ec576c728f8ddb39e8"}, + {file = "wrapt-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ddf582a95641b9a8c8bd643e83f34ecbbfe1b68bc3850093605e469ab680ae3"}, + {file = "wrapt-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:fc5c500966bf48913f795f1984704e6d452ba2414207b15e1f8c339a059d5b16"}, + {file = "wrapt-2.1.1-cp314-cp314-win32.whl", hash = "sha256:4aa4baadb1f94b71151b8e44a0c044f6af37396c3b8bcd474b78b49e2130a23b"}, + {file = "wrapt-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:860e9d3fd81816a9f4e40812f28be4439ab01f260603c749d14be3c0a1170d19"}, + {file = "wrapt-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:3c59e103017a2c1ea0ddf589cbefd63f91081d7ce9d491d69ff2512bb1157e23"}, + {file = "wrapt-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9fa7c7e1bee9278fc4f5dd8275bc8d25493281a8ec6c61959e37cc46acf02007"}, + {file = "wrapt-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:39c35e12e8215628984248bd9c8897ce0a474be2a773db207eb93414219d8469"}, + {file = "wrapt-2.1.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:94ded4540cac9125eaa8ddf5f651a7ec0da6f5b9f248fe0347b597098f8ec14c"}, + {file = "wrapt-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da0af328373f97ed9bdfea24549ac1b944096a5a71b30e41c9b8b53ab3eec04a"}, + {file = "wrapt-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:4ad839b55f0bf235f8e337ce060572d7a06592592f600f3a3029168e838469d3"}, + {file = "wrapt-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0d89c49356e5e2a50fa86b40e0510082abcd0530f926cbd71cf25bee6b9d82d7"}, + {file = "wrapt-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:f4c7dd22cf7f36aafe772f3d88656559205c3af1b7900adfccb70edeb0d2abc4"}, + {file = "wrapt-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f76bc12c583ab01e73ba0ea585465a41e48d968f6d1311b4daec4f8654e356e3"}, + {file = "wrapt-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:7ea74fc0bec172f1ae5f3505b6655c541786a5cabe4bbc0d9723a56ac32eb9b9"}, + {file = "wrapt-2.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e03b3d486eb39f5d3f562839f59094dcee30c4039359ea15768dc2214d9e07c"}, + {file = "wrapt-2.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0fdf3073f488ce4d929929b7799e3b8c52b220c9eb3f4a5a51e2dc0e8ff07881"}, + {file = "wrapt-2.1.1-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0cb4f59238c6625fae2eeb72278da31c9cfba0ff4d9cbe37446b73caa0e9bcf7"}, + {file = "wrapt-2.1.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7f794a1c148871b714cb566f5466ec8288e0148a1c417550983864b3981737cd"}, + {file = "wrapt-2.1.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:95ef3866631c6da9ce1fc0f1e17b90c4c0aa6d041fc70a11bc90733aee122e1a"}, + {file = "wrapt-2.1.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:66bc1b2446f01cbbd3c56b79a3a8435bcd4178ac4e06b091913f7751a7f528b8"}, + {file = "wrapt-2.1.1-cp39-cp39-win32.whl", hash = "sha256:1b9e08e57cabc32972f7c956d10e85093c5da9019faa24faf411e7dd258e528c"}, + {file = "wrapt-2.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:e75ad48c3cca739f580b5e14c052993eb644c7fa5b4c90aa51193280b30875ae"}, + {file = "wrapt-2.1.1-cp39-cp39-win_arm64.whl", hash = "sha256:9ccd657873b7f964711447d004563a2bc08d1476d7a1afcad310f3713e6f50f4"}, + {file = "wrapt-2.1.1-py3-none-any.whl", hash = "sha256:3b0f4629eb954394a3d7c7a1c8cca25f0b07cefe6aa8545e862e9778152de5b7"}, + {file = "wrapt-2.1.1.tar.gz", hash = "sha256:5fdcb09bf6db023d88f312bd0767594b414655d58090fc1c46b3414415f67fac"}, +] + +[package.extras] +dev = ["pytest", "setuptools"] [[package]] name = "xgrammar" @@ -10918,7 +8710,7 @@ description = "Efficient, Flexible and Portable Structured Generation" optional = true python-versions = "<4,>=3.8" groups = ["main"] -markers = "(extra == \"rl\" or extra == \"async-rl\") and (platform_machine == \"x86_64\" or platform_machine == \"aarch64\" or platform_machine == \"arm64\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\")" +markers = "extra == \"vllm\" and (platform_machine == \"x86_64\" or platform_machine == \"aarch64\" or platform_machine == \"arm64\" or platform_machine == \"s390x\" or platform_machine == \"ppc64le\")" files = [ {file = "xgrammar-0.1.29-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:fdc66e834b915cf956168ac086bd577f138261644b944e73d73f07085682a4d8"}, {file = "xgrammar-0.1.29-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:48c5a5c60c5ca5ab09ff5ef9f6b382384a04b153bae5908006cd4f7d80d71e07"}, @@ -11251,18 +9043,6 @@ idna = ">=2.0" multidict = ">=4.0" propcache = ">=0.2.1" -[[package]] -name = "zhconv" -version = "1.4.3" -description = "A simple implementation of Simplified-Traditional Chinese conversion." -optional = true -python-versions = "*" -groups = ["main"] -markers = "extra == \"eval\"" -files = [ - {file = "zhconv-1.4.3.tar.gz", hash = "sha256:ad42d9057ca0605f8e41d62b67ca797f879f58193ee6840562c51459b2698c45"}, -] - [[package]] name = "zipp" version = "3.23.0" @@ -11270,7 +9050,7 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = true python-versions = ">=3.9" groups = ["main"] -markers = "(python_version >= \"3.11\" or extra == \"eval\" or extra == \"megatron\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\") and (python_version == \"3.11\" or extra == \"eval\" or extra == \"megatron\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\" or extra == \"client\") and (extra == \"client\" or extra == \"rl\" or extra == \"async-rl\" or extra == \"server\" or extra == \"eval\" or extra == \"megatron\")" +markers = "extra == \"megatron\" or extra == \"ray\"" files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, @@ -11284,131 +9064,16 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] -[[package]] -name = "zstandard" -version = "0.25.0" -description = "Zstandard bindings for Python" -optional = true -python-versions = ">=3.9" -groups = ["main"] -markers = "python_version >= \"3.11\" and extra == \"client\"" -files = [ - {file = "zstandard-0.25.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e59fdc271772f6686e01e1b3b74537259800f57e24280be3f29c8a0deb1904dd"}, - {file = "zstandard-0.25.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4d441506e9b372386a5271c64125f72d5df6d2a8e8a2a45a0ae09b03cb781ef7"}, - {file = "zstandard-0.25.0-cp310-cp310-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:ab85470ab54c2cb96e176f40342d9ed41e58ca5733be6a893b730e7af9c40550"}, - {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e05ab82ea7753354bb054b92e2f288afb750e6b439ff6ca78af52939ebbc476d"}, - {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:78228d8a6a1c177a96b94f7e2e8d012c55f9c760761980da16ae7546a15a8e9b"}, - {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:2b6bd67528ee8b5c5f10255735abc21aa106931f0dbaf297c7be0c886353c3d0"}, - {file = "zstandard-0.25.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4b6d83057e713ff235a12e73916b6d356e3084fd3d14ced499d84240f3eecee0"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9174f4ed06f790a6869b41cba05b43eeb9a35f8993c4422ab853b705e8112bbd"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:25f8f3cd45087d089aef5ba3848cd9efe3ad41163d3400862fb42f81a3a46701"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3756b3e9da9b83da1796f8809dd57cb024f838b9eeafde28f3cb472012797ac1"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:81dad8d145d8fd981b2962b686b2241d3a1ea07733e76a2f15435dfb7fb60150"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a5a419712cf88862a45a23def0ae063686db3d324cec7edbe40509d1a79a0aab"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:e7360eae90809efd19b886e59a09dad07da4ca9ba096752e61a2e03c8aca188e"}, - {file = "zstandard-0.25.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:75ffc32a569fb049499e63ce68c743155477610532da1eb38e7f24bf7cd29e74"}, - {file = "zstandard-0.25.0-cp310-cp310-win32.whl", hash = "sha256:106281ae350e494f4ac8a80470e66d1fe27e497052c8d9c3b95dc4cf1ade81aa"}, - {file = "zstandard-0.25.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea9d54cc3d8064260114a0bbf3479fc4a98b21dffc89b3459edd506b69262f6e"}, - {file = "zstandard-0.25.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:933b65d7680ea337180733cf9e87293cc5500cc0eb3fc8769f4d3c88d724ec5c"}, - {file = "zstandard-0.25.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a3f79487c687b1fc69f19e487cd949bf3aae653d181dfb5fde3bf6d18894706f"}, - {file = "zstandard-0.25.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:0bbc9a0c65ce0eea3c34a691e3c4b6889f5f3909ba4822ab385fab9057099431"}, - {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:01582723b3ccd6939ab7b3a78622c573799d5d8737b534b86d0e06ac18dbde4a"}, - {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:5f1ad7bf88535edcf30038f6919abe087f606f62c00a87d7e33e7fc57cb69fcc"}, - {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:06acb75eebeedb77b69048031282737717a63e71e4ae3f77cc0c3b9508320df6"}, - {file = "zstandard-0.25.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9300d02ea7c6506f00e627e287e0492a5eb0371ec1670ae852fefffa6164b072"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfd06b1c5584b657a2892a6014c2f4c20e0db0208c159148fa78c65f7e0b0277"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f373da2c1757bb7f1acaf09369cdc1d51d84131e50d5fa9863982fd626466313"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6c0e5a65158a7946e7a7affa6418878ef97ab66636f13353b8502d7ea03c8097"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c8e167d5adf59476fa3e37bee730890e389410c354771a62e3c076c86f9f7778"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:98750a309eb2f020da61e727de7d7ba3c57c97cf6213f6f6277bb7fb42a8e065"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:22a086cff1b6ceca18a8dd6096ec631e430e93a8e70a9ca5efa7561a00f826fa"}, - {file = "zstandard-0.25.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:72d35d7aa0bba323965da807a462b0966c91608ef3a48ba761678cb20ce5d8b7"}, - {file = "zstandard-0.25.0-cp311-cp311-win32.whl", hash = "sha256:f5aeea11ded7320a84dcdd62a3d95b5186834224a9e55b92ccae35d21a8b63d4"}, - {file = "zstandard-0.25.0-cp311-cp311-win_amd64.whl", hash = "sha256:daab68faadb847063d0c56f361a289c4f268706b598afbf9ad113cbe5c38b6b2"}, - {file = "zstandard-0.25.0-cp311-cp311-win_arm64.whl", hash = "sha256:22a06c5df3751bb7dc67406f5374734ccee8ed37fc5981bf1ad7041831fa1137"}, - {file = "zstandard-0.25.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7b3c3a3ab9daa3eed242d6ecceead93aebbb8f5f84318d82cee643e019c4b73b"}, - {file = "zstandard-0.25.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:913cbd31a400febff93b564a23e17c3ed2d56c064006f54efec210d586171c00"}, - {file = "zstandard-0.25.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:011d388c76b11a0c165374ce660ce2c8efa8e5d87f34996aa80f9c0816698b64"}, - {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dffecc361d079bb48d7caef5d673c88c8988d3d33fb74ab95b7ee6da42652ea"}, - {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:7149623bba7fdf7e7f24312953bcf73cae103db8cae49f8154dd1eadc8a29ecb"}, - {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:6a573a35693e03cf1d67799fd01b50ff578515a8aeadd4595d2a7fa9f3ec002a"}, - {file = "zstandard-0.25.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5a56ba0db2d244117ed744dfa8f6f5b366e14148e00de44723413b2f3938a902"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:10ef2a79ab8e2974e2075fb984e5b9806c64134810fac21576f0668e7ea19f8f"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:aaf21ba8fb76d102b696781bddaa0954b782536446083ae3fdaa6f16b25a1c4b"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1869da9571d5e94a85a5e8d57e4e8807b175c9e4a6294e3b66fa4efb074d90f6"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:809c5bcb2c67cd0ed81e9229d227d4ca28f82d0f778fc5fea624a9def3963f91"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f27662e4f7dbf9f9c12391cb37b4c4c3cb90ffbd3b1fb9284dadbbb8935fa708"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:99c0c846e6e61718715a3c9437ccc625de26593fea60189567f0118dc9db7512"}, - {file = "zstandard-0.25.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:474d2596a2dbc241a556e965fb76002c1ce655445e4e3bf38e5477d413165ffa"}, - {file = "zstandard-0.25.0-cp312-cp312-win32.whl", hash = "sha256:23ebc8f17a03133b4426bcc04aabd68f8236eb78c3760f12783385171b0fd8bd"}, - {file = "zstandard-0.25.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffef5a74088f1e09947aecf91011136665152e0b4b359c42be3373897fb39b01"}, - {file = "zstandard-0.25.0-cp312-cp312-win_arm64.whl", hash = "sha256:181eb40e0b6a29b3cd2849f825e0fa34397f649170673d385f3598ae17cca2e9"}, - {file = "zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94"}, - {file = "zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1"}, - {file = "zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f"}, - {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea"}, - {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e"}, - {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551"}, - {file = "zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98"}, - {file = "zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf"}, - {file = "zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09"}, - {file = "zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5"}, - {file = "zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049"}, - {file = "zstandard-0.25.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:e29f0cf06974c899b2c188ef7f783607dbef36da4c242eb6c82dcd8b512855e3"}, - {file = "zstandard-0.25.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:05df5136bc5a011f33cd25bc9f506e7426c0c9b3f9954f056831ce68f3b6689f"}, - {file = "zstandard-0.25.0-cp314-cp314-manylinux2010_i686.manylinux_2_12_i686.manylinux_2_28_i686.whl", hash = "sha256:f604efd28f239cc21b3adb53eb061e2a205dc164be408e553b41ba2ffe0ca15c"}, - {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:223415140608d0f0da010499eaa8ccdb9af210a543fac54bce15babbcfc78439"}, - {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2e54296a283f3ab5a26fc9b8b5d4978ea0532f37b231644f367aa588930aa043"}, - {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ca54090275939dc8ec5dea2d2afb400e0f83444b2fc24e07df7fdef677110859"}, - {file = "zstandard-0.25.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e09bb6252b6476d8d56100e8147b803befa9a12cea144bbe629dd508800d1ad0"}, - {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a9ec8c642d1ec73287ae3e726792dd86c96f5681eb8df274a757bf62b750eae7"}, - {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a4089a10e598eae6393756b036e0f419e8c1d60f44a831520f9af41c14216cf2"}, - {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f67e8f1a324a900e75b5e28ffb152bcac9fbed1cc7b43f99cd90f395c4375344"}, - {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:9654dbc012d8b06fc3d19cc825af3f7bf8ae242226df5f83936cb39f5fdc846c"}, - {file = "zstandard-0.25.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4203ce3b31aec23012d3a4cf4a2ed64d12fea5269c49aed5e4c3611b938e4088"}, - {file = "zstandard-0.25.0-cp314-cp314-win32.whl", hash = "sha256:da469dc041701583e34de852d8634703550348d5822e66a0c827d39b05365b12"}, - {file = "zstandard-0.25.0-cp314-cp314-win_amd64.whl", hash = "sha256:c19bcdd826e95671065f8692b5a4aa95c52dc7a02a4c5a0cac46deb879a017a2"}, - {file = "zstandard-0.25.0-cp314-cp314-win_arm64.whl", hash = "sha256:d7541afd73985c630bafcd6338d2518ae96060075f9463d7dc14cfb33514383d"}, - {file = "zstandard-0.25.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b9af1fe743828123e12b41dd8091eca1074d0c1569cc42e6e1eee98027f2bbd0"}, - {file = "zstandard-0.25.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:4b14abacf83dfb5c25eb4e4a79520de9e7e205f72c9ee7702f91233ae57d33a2"}, - {file = "zstandard-0.25.0-cp39-cp39-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:a51ff14f8017338e2f2e5dab738ce1ec3b5a851f23b18c1ae1359b1eecbee6df"}, - {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3b870ce5a02d4b22286cf4944c628e0f0881b11b3f14667c1d62185a99e04f53"}, - {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:05353cef599a7b0b98baca9b068dd36810c3ef0f42bf282583f438caf6ddcee3"}, - {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:19796b39075201d51d5f5f790bf849221e58b48a39a5fc74837675d8bafc7362"}, - {file = "zstandard-0.25.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:53e08b2445a6bc241261fea89d065536f00a581f02535f8122eba42db9375530"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1f3689581a72eaba9131b1d9bdbfe520ccd169999219b41000ede2fca5c1bfdb"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d8c56bb4e6c795fc77d74d8e8b80846e1fb8292fc0b5060cd8131d522974b751"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:53f94448fe5b10ee75d246497168e5825135d54325458c4bfffbaafabcc0a577"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:c2ba942c94e0691467ab901fc51b6f2085ff48f2eea77b1a48240f011e8247c7"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:07b527a69c1e1c8b5ab1ab14e2afe0675614a09182213f21a0717b62027b5936"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:51526324f1b23229001eb3735bc8c94f9c578b1bd9e867a0a646a3b17109f388"}, - {file = "zstandard-0.25.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:89c4b48479a43f820b749df49cd7ba2dbc2b1b78560ecb5ab52985574fd40b27"}, - {file = "zstandard-0.25.0-cp39-cp39-win32.whl", hash = "sha256:1cd5da4d8e8ee0e88be976c294db744773459d51bb32f707a0f166e5ad5c8649"}, - {file = "zstandard-0.25.0-cp39-cp39-win_amd64.whl", hash = "sha256:37daddd452c0ffb65da00620afb8e17abd4adaae6ce6310702841760c2c26860"}, - {file = "zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b"}, -] - -[package.extras] -cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and python_version < \"3.14\"", "cffi (>=2.0.0b) ; platform_python_implementation != \"PyPy\" and python_version >= \"3.14\""] - [extras] -async-rl = ["TransferQueue", "math-verify", "ray", "swanlab", "vllm"] -client = ["httpx", "openai", "plotext", "textual", "tinker"] -data = ["py-data-juicer"] docs = ["docutils", "myst_parser", "recommonmark", "sphinx", "sphinx-book-theme", "sphinx-copybutton", "sphinx-rtd-theme", "sphinx_markdown_tables", "sphinxcontrib-mermaid"] -eval = ["evalscope"] -megatron = ["mcore_bridge", "megatron-core", "transformer-engine"] -rl = ["ray", "vllm"] -server = ["opentelemetry-api", "opentelemetry-exporter-otlp", "opentelemetry-instrumentation-logging", "opentelemetry-sdk", "psutil", "pynvml", "redis"] -test = ["hypothesis", "pytest", "pytest-asyncio"] +kernels = ["kernels"] +megatron = ["megatron-core", "transformer-engine"] +ray = ["ray"] +tinker = ["tinker"] +transformers = ["accelerate", "torch", "torchvision"] +vllm = ["vllm"] [metadata] lock-version = "2.1" -python-versions = ">=3.10,<=3.13" -content-hash = "2d8b26118c2aa4ef455e5e6983c89df8abc28279fb59146b867ccb9d31441b02" +python-versions = ">=3.11,<3.13" +content-hash = "6bc839d412edaa773717488aaf25cb1e5b663f305e3f9945d9f92fe1524160d2" diff --git a/pyproject.toml b/pyproject.toml index 3641f275..4b52c9c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ client = [ "plotext>=5.2.0", "openai>=1.0.0", "httpx>=0.25.0", - "tinker==0.29.0; python_version >= '3.11'", + "tinker==0.16.1", ] server = [ "redis>=5.0", diff --git a/src/twinkle/server/sampler/tinker_handlers.py b/src/twinkle/server/sampler/tinker_handlers.py index 02cef849..10b636db 100644 --- a/src/twinkle/server/sampler/tinker_handlers.py +++ b/src/twinkle/server/sampler/tinker_handlers.py @@ -8,7 +8,6 @@ import os import traceback -import uuid from collections.abc import Callable from fastapi import Depends, FastAPI, Request from tinker import types @@ -29,17 +28,16 @@ def _sampled_sequence(*, stop_reason, tokens, logprobs): return types.SampledSequence( stop_reason=stop_reason, - sequence_id=uuid.uuid4().hex, - _tokens_list=tokens, - _logprobs_list=logprobs, + tokens=tokens, + logprobs=logprobs, ) def _sample_response(*, sequences, prompt_logprobs, topk_prompt_logprobs): return types.SampleResponse( sequences=sequences, - _prompt_logprobs_list=prompt_logprobs, - _topk_prompt_logprobs_list=topk_prompt_logprobs, + prompt_logprobs=prompt_logprobs, + topk_prompt_logprobs=topk_prompt_logprobs, ) diff --git a/tests/server/contract/client_api_baseline.json b/tests/server/contract/client_api_baseline.json index 243a0567..44672f77 100644 --- a/tests/server/contract/client_api_baseline.json +++ b/tests/server/contract/client_api_baseline.json @@ -417,27 +417,6 @@ "required": true, "schema": { "$defs": { - "DmelChunk": { - "additionalProperties": false, - "properties": { - "dmel": { - "format": "binary", - "title": "Dmel", - "type": "string" - }, - "type": { - "const": "dmel", - "default": "dmel", - "title": "Type", - "type": "string" - } - }, - "required": [ - "dmel" - ], - "title": "DmelChunk", - "type": "object" - }, "EncodedTextChunk": { "additionalProperties": false, "properties": { @@ -558,9 +537,6 @@ }, { "$ref": "#/$defs/ImageChunk" - }, - { - "$ref": "#/$defs/DmelChunk" } ] }, @@ -690,18 +666,6 @@ "default": null, "title": "Prompt Logprobs" }, - "record_stability_info": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Record Stability Info" - }, "sampling_params": { "$ref": "#/$defs/SamplingParams" }, @@ -734,11 +698,6 @@ "title": "Topk Prompt Logprobs", "type": "integer" }, - "topk_sample_logprobs": { - "default": 0, - "title": "Topk Sample Logprobs", - "type": "integer" - }, "type": { "const": "sample", "default": "sample", @@ -1112,30 +1071,59 @@ "name": "body", "required": true, "schema": { - "properties": { - "forward_input": { + "$defs": { + "Datum": { + "additionalProperties": false, + "properties": { + "loss_fn_inputs": { + "additionalProperties": { + "$ref": "#/$defs/TensorData" + }, + "title": "Loss Fn Inputs", + "type": "object" + }, + "model_input": { + "$ref": "#/$defs/ModelInput" + } + }, + "required": [ + "loss_fn_inputs", + "model_input" + ], + "title": "Datum", + "type": "object" + }, + "EncodedTextChunk": { + "additionalProperties": false, + "properties": { + "tokens": { + "items": { + "type": "integer" + }, + "title": "Tokens", + "type": "array" + }, + "type": { + "const": "encoded_text", + "default": "encoded_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "tokens" + ], + "title": "EncodedTextChunk", + "type": "object" + }, + "ForwardBackwardInput": { + "additionalProperties": false, "properties": { "data": { "items": { - "properties": { - "loss_fn_input_spans": { - "pythonType": "'Optional[ProvenanceSpans]'" - }, - "loss_fn_inputs": { - "pythonType": "'LossFnInputs'" - }, - "model_input": { - "pythonType": "'ModelInput'" - }, - "model_input_spans": { - "pythonType": "'Optional[ProvenanceSpans]'" - } - }, - "required": [ - "model_input" - ], - "type": "object" + "$ref": "#/$defs/Datum" }, + "title": "Data", "type": "array" }, "loss_fn": { @@ -1145,36 +1133,202 @@ "ppo", "cispo", "dro" - ] + ], + "title": "Loss Fn", + "type": "string" }, "loss_fn_config": { "anyOf": [ { "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string" - } - ] + "type": "number" }, "type": "object" }, { "type": "null" } - ] + ], + "default": null, + "title": "Loss Fn Config" } }, "required": [ "data", "loss_fn" ], + "title": "ForwardBackwardInput", + "type": "object" + }, + "ImageAssetPointerChunk": { + "additionalProperties": false, + "properties": { + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" + }, + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "location": { + "title": "Location", + "type": "string" + }, + "type": { + "const": "image_asset_pointer", + "default": "image_asset_pointer", + "title": "Type", + "type": "string" + } + }, + "required": [ + "format", + "location" + ], + "title": "ImageAssetPointerChunk", + "type": "object" + }, + "ImageChunk": { + "additionalProperties": false, + "properties": { + "data": { + "format": "binary", + "title": "Data", + "type": "string" + }, + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" + }, + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "type": { + "const": "image", + "default": "image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "data", + "format" + ], + "title": "ImageChunk", + "type": "object" + }, + "ModelInput": { + "additionalProperties": false, + "properties": { + "chunks": { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/EncodedTextChunk" + }, + { + "$ref": "#/$defs/ImageAssetPointerChunk" + }, + { + "$ref": "#/$defs/ImageChunk" + } + ] + }, + "title": "Chunks", + "type": "array" + } + }, + "required": [ + "chunks" + ], + "title": "ModelInput", + "type": "object" + }, + "TensorData": { + "additionalProperties": false, + "properties": { + "data": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "items": { + "type": "number" + }, + "type": "array" + } + ], + "title": "Data" + }, + "dtype": { + "enum": [ + "int64", + "float32" + ], + "title": "Dtype", + "type": "string" + }, + "shape": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Shape" + } + }, + "required": [ + "data", + "dtype" + ], + "title": "TensorData", "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "forward_input": { + "$ref": "#/$defs/ForwardBackwardInput" }, "model_id": { + "title": "Model Id", "type": "string" }, "seq_id": { @@ -1185,13 +1339,16 @@ { "type": "null" } - ] + ], + "default": null, + "title": "Seq Id" } }, "required": [ "forward_input", "model_id" ], + "title": "ForwardRequest", "type": "object" } } @@ -1213,31 +1370,60 @@ "name": "body", "required": true, "schema": { - "properties": { - "forward_backward_input": { + "$defs": { + "Datum": { + "additionalProperties": false, "properties": { - "data": { - "items": { - "properties": { - "loss_fn_input_spans": { - "pythonType": "'Optional[ProvenanceSpans]'" - }, - "loss_fn_inputs": { - "pythonType": "'LossFnInputs'" - }, - "model_input": { - "pythonType": "'ModelInput'" - }, - "model_input_spans": { - "pythonType": "'Optional[ProvenanceSpans]'" - } - }, - "required": [ - "model_input" - ], - "type": "object" + "loss_fn_inputs": { + "additionalProperties": { + "$ref": "#/$defs/TensorData" }, - "type": "array" + "title": "Loss Fn Inputs", + "type": "object" + }, + "model_input": { + "$ref": "#/$defs/ModelInput" + } + }, + "required": [ + "loss_fn_inputs", + "model_input" + ], + "title": "Datum", + "type": "object" + }, + "EncodedTextChunk": { + "additionalProperties": false, + "properties": { + "tokens": { + "items": { + "type": "integer" + }, + "title": "Tokens", + "type": "array" + }, + "type": { + "const": "encoded_text", + "default": "encoded_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "tokens" + ], + "title": "EncodedTextChunk", + "type": "object" + }, + "ForwardBackwardInput": { + "additionalProperties": false, + "properties": { + "data": { + "items": { + "$ref": "#/$defs/Datum" + }, + "title": "Data", + "type": "array" }, "loss_fn": { "enum": [ @@ -1246,36 +1432,202 @@ "ppo", "cispo", "dro" - ] + ], + "title": "Loss Fn", + "type": "string" }, "loss_fn_config": { "anyOf": [ { "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string" - } - ] + "type": "number" }, "type": "object" }, { "type": "null" } - ] + ], + "default": null, + "title": "Loss Fn Config" } }, "required": [ "data", "loss_fn" ], + "title": "ForwardBackwardInput", + "type": "object" + }, + "ImageAssetPointerChunk": { + "additionalProperties": false, + "properties": { + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" + }, + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "location": { + "title": "Location", + "type": "string" + }, + "type": { + "const": "image_asset_pointer", + "default": "image_asset_pointer", + "title": "Type", + "type": "string" + } + }, + "required": [ + "format", + "location" + ], + "title": "ImageAssetPointerChunk", + "type": "object" + }, + "ImageChunk": { + "additionalProperties": false, + "properties": { + "data": { + "format": "binary", + "title": "Data", + "type": "string" + }, + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" + }, + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "type": { + "const": "image", + "default": "image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "data", + "format" + ], + "title": "ImageChunk", + "type": "object" + }, + "ModelInput": { + "additionalProperties": false, + "properties": { + "chunks": { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/EncodedTextChunk" + }, + { + "$ref": "#/$defs/ImageAssetPointerChunk" + }, + { + "$ref": "#/$defs/ImageChunk" + } + ] + }, + "title": "Chunks", + "type": "array" + } + }, + "required": [ + "chunks" + ], + "title": "ModelInput", + "type": "object" + }, + "TensorData": { + "additionalProperties": false, + "properties": { + "data": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "items": { + "type": "number" + }, + "type": "array" + } + ], + "title": "Data" + }, + "dtype": { + "enum": [ + "int64", + "float32" + ], + "title": "Dtype", + "type": "string" + }, + "shape": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Shape" + } + }, + "required": [ + "data", + "dtype" + ], + "title": "TensorData", "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "forward_backward_input": { + "$ref": "#/$defs/ForwardBackwardInput" }, "model_id": { + "title": "Model Id", "type": "string" }, "seq_id": { @@ -1286,13 +1638,16 @@ { "type": "null" } - ] + ], + "default": null, + "title": "Seq Id" } }, "required": [ "forward_backward_input", "model_id" ], + "title": "ForwardBackwardRequest", "type": "object" } } @@ -1358,18 +1713,6 @@ "SupportedModel": { "description": "Information about a model supported by the server.", "properties": { - "max_context_length": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Max Context Length" - }, "model_name": { "anyOf": [ { @@ -1381,30 +1724,6 @@ ], "default": null, "title": "Model Name" - }, - "sampleable": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sampleable" - }, - "trainable": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Trainable" } }, "title": "SupportedModel", @@ -1466,107 +1785,39 @@ "schema": { "additionalProperties": false, "properties": { - "base_model": { + "model_id": { + "title": "Model Id", + "type": "string" + }, + "optimizer": { + "title": "Optimizer", + "type": "boolean" + }, + "path": { + "title": "Path", + "type": "string" + }, + "seq_id": { "anyOf": [ { - "type": "string" + "type": "integer" }, { "type": "null" } ], "default": null, - "title": "Base Model" - }, - "model_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Id" - }, - "model_seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Seq Id" - }, - "optimizer": { - "title": "Optimizer", - "type": "boolean" - }, - "path": { - "title": "Path", - "type": "string" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - }, - "session_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Session Id" + "title": "Seq Id" }, "type": { "const": "load_weights", "default": "load_weights", "title": "Type", "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" - }, - "weights_access_token": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Weights Access Token" } }, "required": [ + "model_id", "path", "optimizer" ], @@ -1743,11 +1994,6 @@ "title": "Model Id", "type": "string" }, - "overwrite": { - "default": false, - "title": "Overwrite", - "type": "boolean" - }, "path": { "anyOf": [ { @@ -1789,21 +2035,6 @@ "default": "save_weights", "title": "Type", "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" } }, "required": [ @@ -1890,21 +2121,6 @@ "default": "save_weights_for_sampler", "title": "Type", "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" } }, "required": [ @@ -2235,10 +2451,6 @@ "title": "Platform", "type": "string" }, - "process_uuid": { - "title": "Process Uuid", - "type": "string" - }, "sdk_version": { "title": "Sdk Version", "type": "string" @@ -2252,8 +2464,7 @@ "events", "platform", "sdk_version", - "session_id", - "process_uuid" + "session_id" ], "title": "TelemetrySendRequest", "type": "object" @@ -2317,8 +2528,7 @@ "checkpoint_type": { "enum": [ "training", - "sampler", - "external" + "sampler" ], "title": "Checkpoint Type", "type": "string" @@ -2361,21 +2571,6 @@ "tinker_path": { "title": "Tinker Path", "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" } }, "required": [ @@ -2549,8 +2744,7 @@ "checkpoint_type": { "enum": [ "training", - "sampler", - "external" + "sampler" ], "title": "Checkpoint Type", "type": "string" @@ -2593,21 +2787,6 @@ "tinker_path": { "title": "Tinker Path", "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" } }, "required": [ @@ -2738,8 +2917,7 @@ "checkpoint_type": { "enum": [ "training", - "sampler", - "external" + "sampler" ], "title": "Checkpoint Type", "type": "string" @@ -2782,21 +2960,6 @@ "tinker_path": { "title": "Tinker Path", "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" } }, "required": [ @@ -2862,7 +3025,7 @@ "statusCode": 200 } }, - "/training_runs/{run_id}/checkpoints/{checkpoint_id:path}": { + "/training_runs/{run_id}/checkpoints/{checkpoint_id}": { "DELETE": { "body": [], "cookies": [], @@ -2890,7 +3053,7 @@ "statusCode": 200 } }, - "/training_runs/{run_id}/checkpoints/{checkpoint_id:path}/publish": { + "/training_runs/{run_id}/checkpoints/{checkpoint_id}/publish": { "POST": { "body": [], "cookies": [], @@ -2956,7 +3119,7 @@ "statusCode": 200 } }, - "/twinkle/checkpoint_path/{run_id}/{checkpoint_id:path}": { + "/twinkle/checkpoint_path/{run_id}/{checkpoint_id}": { "GET": { "body": [], "cookies": [], @@ -3692,7 +3855,7 @@ "statusCode": 200 } }, - "/twinkle/training_runs/{run_id}/checkpoints/{checkpoint_id:path}": { + "/twinkle/training_runs/{run_id}/checkpoints/{checkpoint_id}": { "DELETE": { "body": [], "cookies": [], @@ -4080,21 +4243,6 @@ "request_id": { "title": "Request Id", "type": "string" - }, - "sample_sequence_ids": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sample Sequence Ids" } }, "required": [ @@ -4114,30 +4262,59 @@ "name": "body", "required": true, "schema": { - "properties": { - "forward_input": { + "$defs": { + "Datum": { + "additionalProperties": false, + "properties": { + "loss_fn_inputs": { + "additionalProperties": { + "$ref": "#/$defs/TensorData" + }, + "title": "Loss Fn Inputs", + "type": "object" + }, + "model_input": { + "$ref": "#/$defs/ModelInput" + } + }, + "required": [ + "loss_fn_inputs", + "model_input" + ], + "title": "Datum", + "type": "object" + }, + "EncodedTextChunk": { + "additionalProperties": false, + "properties": { + "tokens": { + "items": { + "type": "integer" + }, + "title": "Tokens", + "type": "array" + }, + "type": { + "const": "encoded_text", + "default": "encoded_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "tokens" + ], + "title": "EncodedTextChunk", + "type": "object" + }, + "ForwardBackwardInput": { + "additionalProperties": false, "properties": { "data": { "items": { - "properties": { - "loss_fn_input_spans": { - "pythonType": "'Optional[ProvenanceSpans]'" - }, - "loss_fn_inputs": { - "pythonType": "'LossFnInputs'" - }, - "model_input": { - "pythonType": "'ModelInput'" - }, - "model_input_spans": { - "pythonType": "'Optional[ProvenanceSpans]'" - } - }, - "required": [ - "model_input" - ], - "type": "object" + "$ref": "#/$defs/Datum" }, + "title": "Data", "type": "array" }, "loss_fn": { @@ -4147,53 +4324,222 @@ "ppo", "cispo", "dro" - ] + ], + "title": "Loss Fn", + "type": "string" }, "loss_fn_config": { "anyOf": [ { "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string" - } - ] + "type": "number" }, "type": "object" }, { "type": "null" } - ] + ], + "default": null, + "title": "Loss Fn Config" } }, "required": [ "data", "loss_fn" ], + "title": "ForwardBackwardInput", "type": "object" }, - "model_id": { - "type": "string" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" + "ImageAssetPointerChunk": { + "additionalProperties": false, + "properties": { + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" }, - { - "type": "null" + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "location": { + "title": "Location", + "type": "string" + }, + "type": { + "const": "image_asset_pointer", + "default": "image_asset_pointer", + "title": "Type", + "type": "string" } - ] + }, + "required": [ + "format", + "location" + ], + "title": "ImageAssetPointerChunk", + "type": "object" + }, + "ImageChunk": { + "additionalProperties": false, + "properties": { + "data": { + "format": "binary", + "title": "Data", + "type": "string" + }, + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" + }, + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "type": { + "const": "image", + "default": "image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "data", + "format" + ], + "title": "ImageChunk", + "type": "object" + }, + "ModelInput": { + "additionalProperties": false, + "properties": { + "chunks": { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/EncodedTextChunk" + }, + { + "$ref": "#/$defs/ImageAssetPointerChunk" + }, + { + "$ref": "#/$defs/ImageChunk" + } + ] + }, + "title": "Chunks", + "type": "array" + } + }, + "required": [ + "chunks" + ], + "title": "ModelInput", + "type": "object" + }, + "TensorData": { + "additionalProperties": false, + "properties": { + "data": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "items": { + "type": "number" + }, + "type": "array" + } + ], + "title": "Data" + }, + "dtype": { + "enum": [ + "int64", + "float32" + ], + "title": "Dtype", + "type": "string" + }, + "shape": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Shape" + } + }, + "required": [ + "data", + "dtype" + ], + "title": "TensorData", + "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "forward_input": { + "$ref": "#/$defs/ForwardBackwardInput" + }, + "model_id": { + "title": "Model Id", + "type": "string" + }, + "seq_id": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Seq Id" } }, "required": [ "forward_input", "model_id" ], + "title": "ForwardRequest", "type": "object" } } @@ -4220,21 +4566,6 @@ "request_id": { "title": "Request Id", "type": "string" - }, - "sample_sequence_ids": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sample Sequence Ids" } }, "required": [ @@ -4254,69 +4585,264 @@ "name": "body", "required": true, "schema": { - "properties": { - "forward_backward_input": { + "$defs": { + "Datum": { + "additionalProperties": false, + "properties": { + "loss_fn_inputs": { + "additionalProperties": { + "$ref": "#/$defs/TensorData" + }, + "title": "Loss Fn Inputs", + "type": "object" + }, + "model_input": { + "$ref": "#/$defs/ModelInput" + } + }, + "required": [ + "loss_fn_inputs", + "model_input" + ], + "title": "Datum", + "type": "object" + }, + "EncodedTextChunk": { + "additionalProperties": false, + "properties": { + "tokens": { + "items": { + "type": "integer" + }, + "title": "Tokens", + "type": "array" + }, + "type": { + "const": "encoded_text", + "default": "encoded_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "tokens" + ], + "title": "EncodedTextChunk", + "type": "object" + }, + "ForwardBackwardInput": { + "additionalProperties": false, "properties": { "data": { "items": { - "properties": { - "loss_fn_input_spans": { - "pythonType": "'Optional[ProvenanceSpans]'" + "$ref": "#/$defs/Datum" + }, + "title": "Data", + "type": "array" + }, + "loss_fn": { + "enum": [ + "cross_entropy", + "importance_sampling", + "ppo", + "cispo", + "dro" + ], + "title": "Loss Fn", + "type": "string" + }, + "loss_fn_config": { + "anyOf": [ + { + "additionalProperties": { + "type": "number" }, - "loss_fn_inputs": { - "pythonType": "'LossFnInputs'" + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Loss Fn Config" + } + }, + "required": [ + "data", + "loss_fn" + ], + "title": "ForwardBackwardInput", + "type": "object" + }, + "ImageAssetPointerChunk": { + "additionalProperties": false, + "properties": { + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" + }, + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "location": { + "title": "Location", + "type": "string" + }, + "type": { + "const": "image_asset_pointer", + "default": "image_asset_pointer", + "title": "Type", + "type": "string" + } + }, + "required": [ + "format", + "location" + ], + "title": "ImageAssetPointerChunk", + "type": "object" + }, + "ImageChunk": { + "additionalProperties": false, + "properties": { + "data": { + "format": "binary", + "title": "Data", + "type": "string" + }, + "expected_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expected Tokens" + }, + "format": { + "enum": [ + "png", + "jpeg" + ], + "title": "Format", + "type": "string" + }, + "type": { + "const": "image", + "default": "image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "data", + "format" + ], + "title": "ImageChunk", + "type": "object" + }, + "ModelInput": { + "additionalProperties": false, + "properties": { + "chunks": { + "items": { + "anyOf": [ + { + "$ref": "#/$defs/EncodedTextChunk" + }, + { + "$ref": "#/$defs/ImageAssetPointerChunk" }, - "model_input": { - "pythonType": "'ModelInput'" + { + "$ref": "#/$defs/ImageChunk" + } + ] + }, + "title": "Chunks", + "type": "array" + } + }, + "required": [ + "chunks" + ], + "title": "ModelInput", + "type": "object" + }, + "TensorData": { + "additionalProperties": false, + "properties": { + "data": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "items": { + "type": "number" }, - "model_input_spans": { - "pythonType": "'Optional[ProvenanceSpans]'" - } - }, - "required": [ - "model_input" - ], - "type": "object" - }, - "type": "array" + "type": "array" + } + ], + "title": "Data" }, - "loss_fn": { + "dtype": { "enum": [ - "cross_entropy", - "importance_sampling", - "ppo", - "cispo", - "dro" - ] + "int64", + "float32" + ], + "title": "Dtype", + "type": "string" }, - "loss_fn_config": { + "shape": { "anyOf": [ { - "additionalProperties": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "string" - } - ] + "items": { + "type": "integer" }, - "type": "object" + "type": "array" }, { "type": "null" } - ] + ], + "default": null, + "title": "Shape" } }, "required": [ "data", - "loss_fn" + "dtype" ], + "title": "TensorData", "type": "object" + } + }, + "additionalProperties": false, + "properties": { + "forward_backward_input": { + "$ref": "#/$defs/ForwardBackwardInput" }, "model_id": { + "title": "Model Id", "type": "string" }, "seq_id": { @@ -4327,13 +4853,16 @@ { "type": "null" } - ] + ], + "default": null, + "title": "Seq Id" } }, "required": [ "forward_backward_input", "model_id" ], + "title": "ForwardBackwardRequest", "type": "object" } } @@ -4360,21 +4889,6 @@ "request_id": { "title": "Request Id", "type": "string" - }, - "sample_sequence_ids": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sample Sequence Ids" } }, "required": [ @@ -4545,41 +5059,9 @@ "schema": { "additionalProperties": false, "properties": { - "base_model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Base Model" - }, "model_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Id" - }, - "model_seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Seq Id" + "title": "Model Id", + "type": "string" }, "optimizer": { "title": "Optimizer", @@ -4601,51 +5083,15 @@ "default": null, "title": "Seq Id" }, - "session_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Session Id" - }, "type": { "const": "load_weights", "default": "load_weights", "title": "Type", "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" - }, - "weights_access_token": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Weights Access Token" } }, "required": [ + "model_id", "path", "optimizer" ], @@ -4676,21 +5122,6 @@ "request_id": { "title": "Request Id", "type": "string" - }, - "sample_sequence_ids": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sample Sequence Ids" } }, "required": [ @@ -4808,21 +5239,6 @@ "request_id": { "title": "Request Id", "type": "string" - }, - "sample_sequence_ids": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sample Sequence Ids" } }, "required": [ @@ -4848,11 +5264,6 @@ "title": "Model Id", "type": "string" }, - "overwrite": { - "default": false, - "title": "Overwrite", - "type": "boolean" - }, "path": { "anyOf": [ { @@ -4894,21 +5305,6 @@ "default": "save_weights", "title": "Type", "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" } }, "required": [ @@ -4941,21 +5337,6 @@ "request_id": { "title": "Request Id", "type": "string" - }, - "sample_sequence_ids": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sample Sequence Ids" } }, "required": [ @@ -5034,21 +5415,6 @@ "default": "save_weights_for_sampler", "title": "Type", "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" } }, "required": [ @@ -5081,21 +5447,6 @@ "request_id": { "title": "Request Id", "type": "string" - }, - "sample_sequence_ids": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sample Sequence Ids" } }, "required": [ @@ -5158,21 +5509,6 @@ "request_id": { "title": "Request Id", "type": "string" - }, - "sample_sequence_ids": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sample Sequence Ids" } }, "required": [ @@ -6948,27 +7284,6 @@ "required": true, "schema": { "$defs": { - "DmelChunk": { - "additionalProperties": false, - "properties": { - "dmel": { - "format": "binary", - "title": "Dmel", - "type": "string" - }, - "type": { - "const": "dmel", - "default": "dmel", - "title": "Type", - "type": "string" - } - }, - "required": [ - "dmel" - ], - "title": "DmelChunk", - "type": "object" - }, "EncodedTextChunk": { "additionalProperties": false, "properties": { @@ -7089,9 +7404,6 @@ }, { "$ref": "#/$defs/ImageChunk" - }, - { - "$ref": "#/$defs/DmelChunk" } ] }, @@ -7221,18 +7533,6 @@ "default": null, "title": "Prompt Logprobs" }, - "record_stability_info": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Record Stability Info" - }, "sampling_params": { "$ref": "#/$defs/SamplingParams" }, @@ -7265,11 +7565,6 @@ "title": "Topk Prompt Logprobs", "type": "integer" }, - "topk_sample_logprobs": { - "default": 0, - "title": "Topk Sample Logprobs", - "type": "integer" - }, "type": { "const": "sample", "default": "sample", @@ -7308,21 +7603,6 @@ "request_id": { "title": "Request Id", "type": "string" - }, - "sample_sequence_ids": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sample Sequence Ids" } }, "required": [ diff --git a/tests/server/contract/client_api_harness.py b/tests/server/contract/client_api_harness.py index 919639bb..59d4ac5a 100644 --- a/tests/server/contract/client_api_harness.py +++ b/tests/server/contract/client_api_harness.py @@ -27,6 +27,7 @@ import dataclasses import json +import re import sys import types as pytypes from collections.abc import Callable, Mapping, Sequence @@ -155,11 +156,11 @@ def _type_contract(annotation: Any, seen: frozenset[str] = frozenset()) -> Any: def _parameter_contract(field: Any) -> dict[str, Any]: - annotation = getattr(field.field_info, 'annotation', field.type_) + field_info = field.field_info return { 'name': field.alias, - 'required': bool(field.required), - 'schema': _type_contract(annotation), + 'required': bool(field_info.is_required()), + 'schema': _type_contract(field_info.annotation), } @@ -188,7 +189,8 @@ def _extract_app_surface(app: FastAPI) -> dict[str, Any]: 'statusCode': route.status_code or 200, } for method in sorted(route.methods & _HTTP_METHODS): - paths.setdefault(route.path, {})[method] = operation + client_path = re.sub(r'{([^}:]+):[^}]+}', r'{\1}', route.path) + paths.setdefault(client_path, {})[method] = operation return {'paths': paths} diff --git a/tests/server/model/test_twinkle_async_inputs.py b/tests/server/model/test_twinkle_async_inputs.py index 7a13b73c..3221394a 100644 --- a/tests/server/model/test_twinkle_async_inputs.py +++ b/tests/server/model/test_twinkle_async_inputs.py @@ -63,6 +63,9 @@ async def schedule_task_and_wait(self, task, **kwargs): self.scheduled.append(kwargs) return await task() + async def call_backend(self, fn, /, *args, admit=True, **kwargs): + return fn(*args, **kwargs) + @pytest.mark.asyncio async def test_forward_backward_resolves_multiple_data_refs_and_field_kwargs() -> None: diff --git a/tests/server/sampler/test_tinker_handlers.py b/tests/server/sampler/test_tinker_handlers.py index 77a15ce6..1c072762 100644 --- a/tests/server/sampler/test_tinker_handlers.py +++ b/tests/server/sampler/test_tinker_handlers.py @@ -75,5 +75,4 @@ async def test_tinker_asample_allows_base_model_session_without_model_path(): assert isinstance(response, types.SampleResponse) assert response.sequences[0].tokens == [1, 2] - assert response.sequences[0].sequence_id assert management.sampler.adapter_paths == [None] diff --git a/tests/server/sampler/test_twinkle_async_rows.py b/tests/server/sampler/test_twinkle_async_rows.py index 30b7016b..234437be 100644 --- a/tests/server/sampler/test_twinkle_async_rows.py +++ b/tests/server/sampler/test_twinkle_async_rows.py @@ -1,5 +1,7 @@ from __future__ import annotations +from types import SimpleNamespace + import pytest from fastapi import FastAPI from starlette.requests import Request @@ -67,6 +69,7 @@ def __init__(self): self.enabled = True self.scheduled = [] self.put_rows = None + self._task_queue_config = SimpleNamespace(effective_execution_timeout=60.0) async def _on_request_start(self, _request): return 'token' @@ -75,6 +78,9 @@ async def schedule_task_and_wait(self, task, **kwargs): self.scheduled.append(kwargs) return await task() + async def call_backend(self, fn, /, *args, admit=True, **kwargs): + return fn(*args, **kwargs) + def submit_generation(self, submission_id, inputs, params, **kwargs): self.submission_id = submission_id self.inputs = inputs From 4bc9bebae8886efd65f857271f8c5cbe825ec662 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Wed, 16 Sep 2026 12:27:55 +0000 Subject: [PATCH 14/32] fix(server): harden tinker GRPO execution guarantees --- src/twinkle/loss/grpo.py | 12 +++-- .../sampler/vllm_sampler/vllm_sampler.py | 8 ++++ src/twinkle/server/sampler/tinker_handlers.py | 10 +++- tests/loss/test_grpo_gkd.py | 21 ++++++++ tests/server/README.md | 27 ----------- tests/server/conftest.py | 11 ++++- tests/server/integration/e2e_helpers.py | 27 +++++++++-- .../integration/test_nccl_safe_tinker_e2e.py | 48 +++++++++++++++---- .../integration/test_nccl_safe_twinkle_e2e.py | 20 +++++++- tests/server/sampler/test_tinker_handlers.py | 3 ++ 10 files changed, 140 insertions(+), 47 deletions(-) delete mode 100644 tests/server/README.md diff --git a/src/twinkle/loss/grpo.py b/src/twinkle/loss/grpo.py index e146edee..2bf8d39f 100644 --- a/src/twinkle/loss/grpo.py +++ b/src/twinkle/loss/grpo.py @@ -202,14 +202,16 @@ def _pad_and_align_to_batch( elif n_sample == n_pos: # Response-only form (e.g. old_logps from vLLM). result[i, pos] = sample - elif n_sample >= seq_len: - # Full-sequence form (e.g. ref_logps right-padded with ignore-value). - result[i, pos] = sample[:seq_len][mask[i]] + elif n_pos == 0 or (n_sample > 0 and pos[-1].item() < n_sample): + # Variable-length full-sequence form. The processor right-pads the + # batch, but per-sample RL fields from Tinker remain unpadded. They + # are valid when every selected mask position exists in this row. + result[i, pos] = sample[pos] else: raise AssertionError(f'data/mask length mismatch at sample {i}: ' f'n_pos={n_pos}, n_sample={n_sample}, seq_len={seq_len} ' - '(expected n_sample == n_pos for response-only form, ' - 'or n_sample >= seq_len for full-sequence form)') + '(expected n_sample == n_pos for response-only form, or all masked positions ' + 'to exist in the per-sample full-sequence form)') return result diff --git a/src/twinkle/sampler/vllm_sampler/vllm_sampler.py b/src/twinkle/sampler/vllm_sampler/vllm_sampler.py index fe68b9d7..6438fa66 100644 --- a/src/twinkle/sampler/vllm_sampler/vllm_sampler.py +++ b/src/twinkle/sampler/vllm_sampler/vllm_sampler.py @@ -251,6 +251,14 @@ async def _sample_single( else: feat['input_ids'] = response.prompt_token_ids feat['labels'] = [-100] * len(response.prompt_token_ids) + # A sampling prompt (e.g. a tinker ModelInput) carries input_ids but no labels; + # concat_input_feature would then derive a zero-length prefix completion_mask + # and raise. The prompt is pure context, so materialise aligned all-context + # labels when they are missing or length-mismatched. Present, aligned labels are + # left untouched (preserving provenance), and the logprobs-only path -- which + # never concatenates -- is not touched. + if not logprobs_only and 'input_ids' in feat and len(feat.get('labels') or []) != len(feat['input_ids']): + feat['labels'] = [-100] * len(feat['input_ids']) sequences = [] for seq in response.sequences: if logprobs_only: diff --git a/src/twinkle/server/sampler/tinker_handlers.py b/src/twinkle/server/sampler/tinker_handlers.py index 10b636db..3f879001 100644 --- a/src/twinkle/server/sampler/tinker_handlers.py +++ b/src/twinkle/server/sampler/tinker_handlers.py @@ -99,6 +99,10 @@ async def _do_sample(): top_p=body.sampling_params.top_p, top_k=body.sampling_params.top_k, stop=body.sampling_params.stop, + # tinker 0.16.1 has no SamplingParams.logprobs field, but its + # SampledSequence contract and GRPO training require one + # chosen-token logprob per generated token. + logprobs=1, ) # A resolved checkpoint is either a LoRA adapter dir (has @@ -131,8 +135,12 @@ async def _do_sample(): flattened = [float(lp_list[0][1]) for lp_list in seq.logprobs if lp_list] except (IndexError, TypeError): flattened = [] - if flattened and len(flattened) == len(seq.logprobs): + if len(flattened) == len(seq.tokens): logprobs = flattened + else: + raise RuntimeError( + f'Sampler returned {len(flattened)} logprobs for {len(seq.tokens)} generated ' + 'tokens; refusing to return a misaligned Tinker SampledSequence.') tinker_sequences.append( _sampled_sequence( stop_reason=seq.stop_reason, diff --git a/tests/loss/test_grpo_gkd.py b/tests/loss/test_grpo_gkd.py index 3d0f5512..1f9daa70 100644 --- a/tests/loss/test_grpo_gkd.py +++ b/tests/loss/test_grpo_gkd.py @@ -82,6 +82,27 @@ def test_grpo_list_advantages(self): result = loss_fn(inputs, outputs, old_logps=old_logps, advantages=adv_list) assert torch.isfinite(result['loss']) + def test_pad_variable_length_full_sequence_rows(self): + """Unpadded per-sample rows align against a right-padded batch mask.""" + mask = torch.tensor([ + [False, True, True, False, False], + [False, False, True, True, True], + ]) + rows = [ + [-9.0, 0.1, 0.2], + [-9.0, -9.0, 0.3, 0.4, 0.5], + ] + + got = GRPOLoss()._pad_and_align_to_batch(rows, mask, mask.device, torch.float32) + + assert torch.equal(got[0], torch.tensor([0.0, 0.1, 0.2, 0.0, 0.0])) + assert torch.equal(got[1], torch.tensor([0.0, 0.0, 0.3, 0.4, 0.5])) + + def test_pad_rejects_full_sequence_missing_a_masked_position(self): + mask = torch.tensor([[False, False, True, True, True]]) + with pytest.raises(AssertionError, match='all masked positions'): + GRPOLoss()._pad_and_align_to_batch([[0.1, 0.2, 0.3, 0.4]], mask, mask.device, torch.float32) + def test_grpo_weights_sequences_equally(self): labels = torch.tensor([ [1, -100, -100], diff --git a/tests/server/README.md b/tests/server/README.md deleted file mode 100644 index 49adf2cb..00000000 --- a/tests/server/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Server test notes - -## Mock backend evidence boundary (spec T8.4 / R9#10) - -`server/model/backends/mock_model.py` is a stand-in backend for tests that must run -without a GPU or a real Ray-distributed model. Two properties bound what a test using -it can prove: - -1. **Every method takes `**kwargs` and performs no argument validation.** A test that - drives the mock backend therefore **cannot** be used as evidence for request/argument - validation behavior — the mock accepts anything. -2. **It never enters a real collective.** The mock does no NCCL communication, so a test - using it **cannot** be used as evidence for NCCL behavior (asymmetric failure, - collective mis-pairing, ReduceScatter, etc.). - -What the mock backend *can* evidence is exactly the parts that do not depend on the -backend's internals: the **dispatch path** (that a call reaches the backend via -`call_backend` / the task queue) and the **timeout / admission mechanisms** themselves -(that a slow or leaked call is bounded and the event loop stays responsive). - -Tests that need to prove validation or NCCL behavior are the GPU-gated end-to-end tests -under `tests/server/integration/test_nccl_safe_*_e2e.py` (run only with -`TWINKLE_TEST_GPU_E2E=1` against a real server configured with the test execution timeout). - -The contract suite covers all five apps and recursively snapshots request and response -types. Tinker compatibility assertions follow the 0.29.0 SDK wire values. The blocking -boundary suite uses a real serial Ray actor for the health-probe timing case. diff --git a/tests/server/conftest.py b/tests/server/conftest.py index 4ac4d24f..fea2e62c 100644 --- a/tests/server/conftest.py +++ b/tests/server/conftest.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Shared Ray runtime + per-test isolation for ``tests/server`` (state, cli, ...). +"""Shared Ray runtime, per-test isolation, and evidence boundaries. ``RayActorBackend`` is a forwarding wrapper around a detached Ray actor; instantiating one without an initialized Ray runtime raises @@ -12,6 +12,15 @@ of the actor wrapper. To keep tests independent we clear that actor's store before each test function. Tests that pin a non-default ``key_prefix`` get their own actor; this fixture intentionally leaves those alone. + +Evidence boundary (spec T8.4 / R9#10): every mock-model backend method accepts +``**kwargs`` without argument validation, and the mock enters no real collective. +A mock-backed test therefore proves neither request/argument validation nor NCCL +behavior (asymmetric failure, collective mis-pairing, ReduceScatter, etc.). It may +prove only backend dispatch, task-queue behavior, timeout/admission mechanisms, +and event-loop responsiveness. Validation and NCCL claims require the GPU-gated +``test_nccl_safe_*_e2e.py`` tests against a real server. The contract suite covers +all five apps; Tinker compatibility follows the pinned 0.16.1 SDK wire values. """ from __future__ import annotations diff --git a/tests/server/integration/e2e_helpers.py b/tests/server/integration/e2e_helpers.py index 9d3073ba..2eca95f0 100644 --- a/tests/server/integration/e2e_helpers.py +++ b/tests/server/integration/e2e_helpers.py @@ -22,7 +22,9 @@ MODEL_ID = f'ms://{BASE_MODEL}' BASE_URL = os.environ.get('TWINKLE_SERVER_URL', 'http://localhost:9000') API_KEY = 'EMPTY_API_KEY' -TIMEOUT = 120 # seconds per operation before declaring hang +TIMEOUT = float(os.environ.get('TWINKLE_TEST_OPERATION_TIMEOUT', '120')) +# Per-operation hang threshold. PPU Megatron cold JIT can exceed 300s; callers may +# raise this without weakening the default CI/GPU bound. GRADIENT_ACCUMULATION_STEPS = 2 # Megatron requires GA >= 2 @@ -66,6 +68,17 @@ def log(msg: str) -> None: # Dataset Factories # ═══════════════════════════════════════════════════════════════════════════ + +def _local_arrow_dataset(path: str, data_slice): + """Load selected rows from cached Arrow without hub metadata access.""" + from datasets import Dataset as HFDataset + from twinkle.dataset import Dataset, DatasetMeta + + source = HFDataset.from_file(path) + indices = [index % len(source) for index in data_slice] + return Dataset(DatasetMeta(data=source.select(indices))) + + def create_sft_dataset(data_slice=range(100)): """Create SelfCognition SFT dataset (small slice for speed).""" from twinkle.dataloader import DataLoader @@ -83,7 +96,11 @@ def create_dpo_dataset(data_slice=range(50)): from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import EmojiDPOProcessor - dataset = Dataset(DatasetMeta('ms://hjh0119/shareAI-Llama3-DPO-zh-en-emoji', data_slice=data_slice)) + local_arrow = os.environ.get('TWINKLE_TEST_DPO_ARROW') + if local_arrow: + dataset = _local_arrow_dataset(local_arrow, data_slice) + else: + dataset = Dataset(DatasetMeta('ms://hjh0119/shareAI-Llama3-DPO-zh-en-emoji', data_slice=data_slice)) dataset.set_template('Qwen3_5Template', model_id=MODEL_ID, max_length=1024) dataset.map(EmojiDPOProcessor, init_args={'system': 'You are a helpful assistant.'}) dataset.encode() @@ -97,7 +114,11 @@ def create_grpo_dataset(data_slice=range(50)): system_prompt = ('You are a helpful math assistant. Solve the problem with minimal but correct reasoning ' 'and put your final answer within \\boxed{}.') - dataset = Dataset(DatasetMeta('ms://modelscope/gsm8k', subset_name='main', split='train', data_slice=data_slice)) + local_arrow = os.environ.get('TWINKLE_TEST_GRPO_ARROW') + if local_arrow: + dataset = _local_arrow_dataset(local_arrow, data_slice) + else: + dataset = Dataset(DatasetMeta('ms://modelscope/gsm8k', subset_name='main', split='train', data_slice=data_slice)) dataset.set_template('Qwen3_5Template', model_id=MODEL_ID, max_length=2048, enable_thinking=False) dataset.map(GSM8KProcessor(system=system_prompt)) dataset.encode(add_generation_prompt=True) diff --git a/tests/server/integration/test_nccl_safe_tinker_e2e.py b/tests/server/integration/test_nccl_safe_tinker_e2e.py index 4846a2bc..155f2738 100644 --- a/tests/server/integration/test_nccl_safe_tinker_e2e.py +++ b/tests/server/integration/test_nccl_safe_tinker_e2e.py @@ -31,6 +31,11 @@ SERVER_URL = os.environ.get('TWINKLE_SERVER_URL', 'http://localhost:9000') EXECUTION_TIMEOUT = float(os.environ.get('TWINKLE_TEST_EXECUTION_TIMEOUT', '30')) TIMEOUT = EXECUTION_TIMEOUT + 15 +# The `global_rank=` attribution is added by `nccl_safe_megatron`, which decorates +# only the Megatron backend; the transformers backend carries no such annotation +# (its former silent-degradation decorator was removed by R6#3). Gate the rank-attribution assertion +# on the backend so this file is safe under TWINKLE_TEST_BACKEND=transformers. +BACKEND = os.environ.get('TWINKLE_TEST_BACKEND', 'megatron') def _init_client(): @@ -83,11 +88,23 @@ def test_failure_is_terminal_then_valid_request_succeeds(): assert caught.value.category is types.RequestErrorCategory.Server assert time.time() - start < TIMEOUT, 'malformed request must fail fast, not hang (NCCL)' - # Recovery: a subsequent valid request on the same deployment must succeed. + # Recovery: megatron commits the DDP reducer inside its fused forward_backward, so + # a subsequent valid request must succeed. The tinker transformers path runs + # forward()/loss/backward() separately; a mid-iteration loss failure leaves DDP's + # reducer half-finished and poisons the next request, so the spec only guarantees a + # *terminal* response there (R6#14), not success. (tinker 0.16.1 exposes no GA knob.) good = [_make_datum() for _ in range(4)] - result = tc.forward_backward(good, 'importance_sampling').result() - assert result is not None - tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() + if BACKEND == 'megatron': + result = tc.forward_backward(good, 'importance_sampling').result() + assert result is not None + tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() + else: + try: + result = tc.forward_backward(good, 'importance_sampling').result(timeout=TIMEOUT) + assert result is not None + tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() + except RequestFailedError as exc: + assert exc.category is types.RequestErrorCategory.Server def test_partial_rank_failure_is_terminal_then_recovers(): @@ -101,9 +118,24 @@ def test_partial_rank_failure_is_terminal_then_recovers(): with pytest.raises(RequestFailedError) as caught: tc.forward_backward(batch, 'importance_sampling').result(timeout=TIMEOUT) assert caught.value.category is types.RequestErrorCategory.Server - assert 'global_rank=' in str(caught.value) + # Megatron attributes the failure to a global rank via nccl_safe_megatron; the + # transformers backend has no such annotation (R6#3 removed its old decorator). + if BACKEND == 'megatron': + assert 'global_rank=' in str(caught.value) assert time.time() - start < TIMEOUT - result = tc.forward_backward([_make_datum() for _ in range(4)], 'importance_sampling').result() - assert result is not None - tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() + # See the recovery note above: success is required only where forward_backward + # commits the DDP reducer atomically (megatron). On transformers a terminal + # loud failure is acceptable (spec R6#14) -- the guarantee is no hang. + if BACKEND == 'megatron': + result = tc.forward_backward([_make_datum() for _ in range(4)], 'importance_sampling').result() + assert result is not None + tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() + else: + try: + result = tc.forward_backward([_make_datum() for _ in range(4)], + 'importance_sampling').result(timeout=TIMEOUT) + assert result is not None + tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() + except RequestFailedError as exc: + assert exc.category is types.RequestErrorCategory.Server diff --git a/tests/server/integration/test_nccl_safe_twinkle_e2e.py b/tests/server/integration/test_nccl_safe_twinkle_e2e.py index 5eed7141..29b89be4 100644 --- a/tests/server/integration/test_nccl_safe_twinkle_e2e.py +++ b/tests/server/integration/test_nccl_safe_twinkle_e2e.py @@ -31,6 +31,12 @@ EXECUTION_TIMEOUT = float(os.environ.get('TWINKLE_TEST_EXECUTION_TIMEOUT', '30')) TIMEOUT = EXECUTION_TIMEOUT + 15 ADAPTER_NAME = 'loud-failure-test' +# The `global_rank=` attribution is added by `nccl_safe_megatron`, which decorates +# only the Megatron backend; the transformers backend's forward_backward carries no +# such annotation (its former silent-degradation decorator was removed by R6#3). Gate the +# rank-attribution assertion on the backend so this file is safe to run under the +# integration-e2e SKILL's TWINKLE_TEST_BACKEND=transformers path. +BACKEND = os.environ.get('TWINKLE_TEST_BACKEND', 'megatron') def _init_client(): @@ -43,7 +49,11 @@ def _init_client(): model.add_adapter_to_model( adapter_name=ADAPTER_NAME, config=LoraConfig(r=16, target_modules=['q_proj', 'v_proj']), - gradient_accumulation_steps=1, + # GA>=2 (repo convention, see e2e_helpers): with GA=1 every backward syncs + # DDP immediately, so a mid-iteration failure can leave the reducer + # half-finished and poison the next request. GA=2 runs accumulation steps + # under no_sync, keeping the recovery request clean. + gradient_accumulation_steps=2, ) model.set_loss('GRPOLoss', init_args={'epsilon': 0.2}) model.set_optimizer('Adam', lr=1e-5) @@ -83,7 +93,13 @@ def test_failure_is_terminal_then_valid_request_succeeds(): with pytest.raises(Exception) as caught: model.forward_backward( inputs=bad_features, adapter_name=ADAPTER_NAME, old_logps=bad_old_logps, advantages=bad_adv) - assert 'global_rank=' in str(caught.value) + message = str(caught.value) + # The failure must be loud and descriptive (not a silent zero-loss success): + # the deliberate old_logps/completion length mismatch surfaces on both backends. + assert 'mismatch' in message, message + # Megatron additionally attributes the failure to a global rank via nccl_safe_megatron. + if BACKEND == 'megatron': + assert 'global_rank=' in message, message assert time.time() - start < TIMEOUT, 'malformed request must fail fast, not hang (NCCL)' good_features, good_old_logps, good_adv = _make_inputs() diff --git a/tests/server/sampler/test_tinker_handlers.py b/tests/server/sampler/test_tinker_handlers.py index 1c072762..e6fe1596 100644 --- a/tests/server/sampler/test_tinker_handlers.py +++ b/tests/server/sampler/test_tinker_handlers.py @@ -20,6 +20,7 @@ class _DummySampler: def __init__(self): self.adapter_paths = [] + self.sampling_params = [] def set_template(self, *args, **kwargs): return None @@ -29,6 +30,7 @@ def reset_prefix_cache(self): def sample(self, inputs, sampling_params=None, adapter_name='', *, adapter_path=None, **kwargs): self.adapter_paths.append(adapter_path) + self.sampling_params.append(sampling_params) return [ SampleResponse( sequences=[SampledSequence( @@ -76,3 +78,4 @@ async def test_tinker_asample_allows_base_model_session_without_model_path(): assert isinstance(response, types.SampleResponse) assert response.sequences[0].tokens == [1, 2] assert management.sampler.adapter_paths == [None] + assert management.sampler.sampling_params[0].logprobs == 1 From fc07df526024c781bad4cd98b24b5321dfba6365 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Wed, 16 Sep 2026 12:48:43 +0000 Subject: [PATCH 15/32] test(server): simplify execution guarantee coverage --- .../integration/test_nccl_safe_tinker_e2e.py | 54 ++++++++----------- tests/server/utils/test_task_errors.py | 11 +--- 2 files changed, 23 insertions(+), 42 deletions(-) diff --git a/tests/server/integration/test_nccl_safe_tinker_e2e.py b/tests/server/integration/test_nccl_safe_tinker_e2e.py index 155f2738..3d84e9fe 100644 --- a/tests/server/integration/test_nccl_safe_tinker_e2e.py +++ b/tests/server/integration/test_nccl_safe_tinker_e2e.py @@ -68,6 +68,23 @@ def _make_datum(seq_len=64, completion_len=32, *, bad_logprobs_len=None): ) +def _assert_recovery_terminal(tc) -> None: + """Require success on Megatron; Transformers may fail loudly after DDP poisoning.""" + from tinker import types + from tinker._exceptions import RequestFailedError + + request = tc.forward_backward([_make_datum() for _ in range(4)], 'importance_sampling') + if BACKEND == 'megatron': + assert request.result() is not None + tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() + return + try: + assert request.result(timeout=TIMEOUT) is not None + tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() + except RequestFailedError as exc: + assert exc.category is types.RequestErrorCategory.Server + + def test_failure_is_terminal_then_valid_request_succeeds(): """A malformed request fails loudly (terminal), a subsequent valid one succeeds. @@ -88,23 +105,10 @@ def test_failure_is_terminal_then_valid_request_succeeds(): assert caught.value.category is types.RequestErrorCategory.Server assert time.time() - start < TIMEOUT, 'malformed request must fail fast, not hang (NCCL)' - # Recovery: megatron commits the DDP reducer inside its fused forward_backward, so - # a subsequent valid request must succeed. The tinker transformers path runs - # forward()/loss/backward() separately; a mid-iteration loss failure leaves DDP's - # reducer half-finished and poisons the next request, so the spec only guarantees a - # *terminal* response there (R6#14), not success. (tinker 0.16.1 exposes no GA knob.) - good = [_make_datum() for _ in range(4)] - if BACKEND == 'megatron': - result = tc.forward_backward(good, 'importance_sampling').result() - assert result is not None - tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() - else: - try: - result = tc.forward_backward(good, 'importance_sampling').result(timeout=TIMEOUT) - assert result is not None - tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() - except RequestFailedError as exc: - assert exc.category is types.RequestErrorCategory.Server + # Megatron must recover successfully. Tinker's Transformers path executes + # forward/loss/backward separately; after a mid-iteration failure, R6#14 only + # guarantees that the next request reaches a terminal state. + _assert_recovery_terminal(tc) def test_partial_rank_failure_is_terminal_then_recovers(): @@ -124,18 +128,4 @@ def test_partial_rank_failure_is_terminal_then_recovers(): assert 'global_rank=' in str(caught.value) assert time.time() - start < TIMEOUT - # See the recovery note above: success is required only where forward_backward - # commits the DDP reducer atomically (megatron). On transformers a terminal - # loud failure is acceptable (spec R6#14) -- the guarantee is no hang. - if BACKEND == 'megatron': - result = tc.forward_backward([_make_datum() for _ in range(4)], 'importance_sampling').result() - assert result is not None - tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() - else: - try: - result = tc.forward_backward([_make_datum() for _ in range(4)], - 'importance_sampling').result(timeout=TIMEOUT) - assert result is not None - tc.optim_step(types.AdamParams(learning_rate=1e-5)).result() - except RequestFailedError as exc: - assert exc.category is types.RequestErrorCategory.Server + _assert_recovery_terminal(tc) diff --git a/tests/server/utils/test_task_errors.py b/tests/server/utils/test_task_errors.py index 832ca109..494dc535 100644 --- a/tests/server/utils/test_task_errors.py +++ b/tests/server/utils/test_task_errors.py @@ -1,4 +1,4 @@ -from twinkle.server.utils.task_errors import error_payload_from_stored, task_error_payload +from twinkle.server.utils.task_errors import task_error_payload from twinkle_client.types.errors import ErrorCategory @@ -26,12 +26,3 @@ def test_error_summary_is_single_line(): payload = task_error_payload( 'RuntimeError: boom\n File "/server/path.py", line 1', request_id='req-lines') assert payload['error'] == 'RuntimeError: boom' - - -def test_error_payload_from_stored_backfills_two_field_legacy(): - stored = {'error': 'boom', 'category': 'Server'} - - payload = error_payload_from_stored(stored, request_id='req_3') - - assert payload.error_code == 500 - assert payload.request_id == 'req_3' From 9ea3272459d705e7362daa9680932bc856b1704d Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Thu, 17 Sep 2026 13:41:21 +0000 Subject: [PATCH 16/32] feat(server): converge twinkle-native requests to submit/retrieve lifecycle Implements the server-request-lifecycle spec (Part 2). A single HTTP request's server-side duration is now decoupled from task execution time: submit enqueues and returns a TaskEnvelope immediately, and the client's future layer polls a dedicated retrieve endpoint. The motivation is narrow and load-bearing: the client's per-request timeout was 600s while most ingress gateways cut idle connections at 60s, so data-plane endpoints (forward_backward, sample) were already unusable behind a real gateway. Every single HTTP request is now bounded by the 30s long-poll window regardless of how long the task runs. This does not improve throughput or training speed -- the compute queue is still serial and GPU utilisation is unchanged. Server: - new twinkle/server/lifecycle/: envelope.py (the one FutureRecord -> TaskEnvelope mapping point), poll_config.py (single declaration of the long-poll window and interval, shared by both retrieve endpoints), submit.py (run_submit shell plus the to_backend_inputs / backend_kwargs / input_metrics seams left for Part 3) - new POST /twinkle/retrieve_future and POST /twinkle/cancel - preflight now raises RequestRejectedError subclasses, so a rejected request returns a real status code and writes zero future records - TwinkleServerError handler puts ErrorPayload fields at the response top level - delete schedule_task_and_wait, run_task (both copies), QueuedTask.completion, _complete_result/_complete_error and persist_status; the future record is now the only delivery channel for results and failures - delete TaskStatus.RATE_LIMITED (limiting is now HTTP 429), the get_state_dict endpoint and the upload_status endpoint Client: - new types/lifecycle.py (TaskEnvelope), _future.py (the only polling loop), exceptions.py (TwinkleHTTPError / TaskFailedError / TaskCancelledError / TaskWaitTimeoutError / TaskRecordLostError) - three separate 600s timeout literals collapse into _HTTP_TIMEOUT = 90 - public methods keep their synchronous signatures and return types, so cookbook scripts and integration tests are unchanged Breaking changes: queued endpoints return TaskEnvelope instead of a business model; task failure raises TaskFailedError (HTTP 200 + payload) instead of requests.HTTPError (HTTP 500); get_state_dict is removed (use save + read the checkpoint). Verified on real PPU hardware (Qwen3.5-4B, 8x ZW810): SFT/DPO/GRPO x twinkle/tinker on the transformers backend and SFT x twinkle/tinker on megatron, 8/8 passing with losses identical to the pre-refactor run. Unit suite: 360 passed, 0 failed. --- .gitignore | 5 + cookbook/client/async_rl/server_config.yaml | 3 +- .../client/server/megatron/server_config.yaml | 3 +- .../server/transformer/server_config.yaml | 6 +- cookbook/client/tinker/upload_to_hub.py | 8 +- cookbook/client/twinkle/upload_to_hub.py | 8 +- .../Usage Guide/Server and Client/Server.md | 11 +- .../\346\234\215\345\212\241\347\253\257.md" | 11 +- src/twinkle/server/config/persistence.py | 7 +- src/twinkle/server/deployment.py | 22 + src/twinkle/server/exceptions.py | 100 +- src/twinkle/server/gateway/tinker_handlers.py | 16 +- .../server/gateway/twinkle_handlers.py | 61 + src/twinkle/server/lifecycle/__init__.py | 7 + src/twinkle/server/lifecycle/envelope.py | 64 + src/twinkle/server/lifecycle/poll_config.py | 79 + src/twinkle/server/lifecycle/submit.py | 150 + src/twinkle/server/model/app.py | 2 +- src/twinkle/server/model/twinkle_handlers.py | 940 +- .../server/sampler/twinkle_handlers.py | 98 +- src/twinkle/server/state/backend/__init__.py | 2 - src/twinkle/server/state/backend/factory.py | 11 +- .../server/state/backend/file_backend.py | 235 - src/twinkle/server/state/future_manager.py | 38 +- src/twinkle/server/state/model_manager.py | 4 +- src/twinkle/server/state/server_state.py | 36 +- src/twinkle/server/utils/task_errors.py | 2 +- src/twinkle/server/utils/task_queue/config.py | 5 + src/twinkle/server/utils/task_queue/mixin.py | 207 +- src/twinkle/server/utils/task_queue/types.py | 10 +- src/twinkle/server/utils/task_queue/worker.py | 94 +- src/twinkle_client/_future.py | 168 + src/twinkle_client/exceptions.py | 116 + src/twinkle_client/http/http_utils.py | 60 +- .../model/multi_lora_transformers.py | 175 +- src/twinkle_client/sampler/vllm_sampler.py | 22 +- src/twinkle_client/types/__init__.py | 5 +- src/twinkle_client/types/lifecycle.py | 83 + src/twinkle_client/types/model.py | 25 +- tests/server/config/test_server_config.py | 11 - .../server/contract/client_api_baseline.json | 8353 ----------------- tests/server/contract/client_api_harness.py | 81 +- tests/server/contract/client_api_routes.json | 628 ++ .../contract/test_client_api_contract.py | 64 +- tests/server/contract/update_baseline.py | 21 +- tests/server/data_plane/test_store.py | 6 +- tests/server/fixtures/server_config_mock.yaml | 7 +- .../integration/test_mock_mode_startup.py | 2 - tests/server/lifecycle/__init__.py | 1 + tests/server/lifecycle/test_envelope.py | 52 + .../lifecycle/test_envelope_coverage.py | 66 + .../lifecycle/test_preflight_rejection.py | 115 + .../lifecycle/test_retrieve_endpoint.py | 86 + .../server/lifecycle/test_run_submit_dedup.py | 102 + tests/server/lifecycle/test_static_guards.py | 111 + .../server/lifecycle/test_submit_peek_e2e.py | 83 + tests/server/lifecycle/test_timing_bounds.py | 139 + .../test_tinker_retrieve_regression.py | 60 + tests/server/model/test_mock_model.py | 1 - .../server/model/test_twinkle_async_inputs.py | 8 +- tests/server/sampler/test_mock_sampler.py | 6 + .../server/sampler/test_twinkle_async_rows.py | 17 +- tests/server/state/fake_backend.py | 76 + tests/server/state/test_future_lifecycle.py | 115 +- tests/server/state/test_update_atomic.py | 49 - tests/server/utils/test_task_queue_mixin.py | 96 +- tests/twinkle_client/test_async_components.py | 15 +- tests/twinkle_client/test_error_parsing.py | 69 + tests/twinkle_client/test_future_layer.py | 137 + 69 files changed, 3720 insertions(+), 9756 deletions(-) create mode 100644 src/twinkle/server/lifecycle/__init__.py create mode 100644 src/twinkle/server/lifecycle/envelope.py create mode 100644 src/twinkle/server/lifecycle/poll_config.py create mode 100644 src/twinkle/server/lifecycle/submit.py delete mode 100644 src/twinkle/server/state/backend/file_backend.py create mode 100644 src/twinkle_client/_future.py create mode 100644 src/twinkle_client/exceptions.py create mode 100644 src/twinkle_client/types/lifecycle.py delete mode 100644 tests/server/contract/client_api_baseline.json create mode 100644 tests/server/contract/client_api_routes.json create mode 100644 tests/server/lifecycle/__init__.py create mode 100644 tests/server/lifecycle/test_envelope.py create mode 100644 tests/server/lifecycle/test_envelope_coverage.py create mode 100644 tests/server/lifecycle/test_preflight_rejection.py create mode 100644 tests/server/lifecycle/test_retrieve_endpoint.py create mode 100644 tests/server/lifecycle/test_run_submit_dedup.py create mode 100644 tests/server/lifecycle/test_static_guards.py create mode 100644 tests/server/lifecycle/test_submit_peek_e2e.py create mode 100644 tests/server/lifecycle/test_timing_bounds.py create mode 100644 tests/server/lifecycle/test_tinker_retrieve_regression.py create mode 100644 tests/server/state/fake_backend.py create mode 100644 tests/twinkle_client/test_error_parsing.py create mode 100644 tests/twinkle_client/test_future_layer.py diff --git a/.gitignore b/.gitignore index ef5e183b..ad9fcd68 100644 --- a/.gitignore +++ b/.gitignore @@ -161,3 +161,8 @@ test_cookbook/ /test*.py swanlog/ tests/server/config/_generated_e2e.yaml + +# Full field-level contract surface: a generated artifact (8k+ lines, unreviewable diff). +# Regenerate via `python -m tests.server.contract.update_baseline`; do not commit it. +# NOTE: client_api_routes.json is the compact, COMMITTED guard -- do not ignore that one. +tests/server/contract/client_api_baseline.json diff --git a/cookbook/client/async_rl/server_config.yaml b/cookbook/client/async_rl/server_config.yaml index 6c0eb612..d627521d 100644 --- a/cookbook/client/async_rl/server_config.yaml +++ b/cookbook/client/async_rl/server_config.yaml @@ -19,8 +19,7 @@ telemetry: otlp_endpoint: http://localhost:4317 persistence: - mode: file - file_path: /tmp/twinkle_state.json + mode: memory applications: diff --git a/cookbook/client/server/megatron/server_config.yaml b/cookbook/client/server/megatron/server_config.yaml index 81d14e75..680e2db8 100644 --- a/cookbook/client/server/megatron/server_config.yaml +++ b/cookbook/client/server/megatron/server_config.yaml @@ -18,10 +18,9 @@ telemetry: # Top-level placement makes the launcher propagate this to every Ray worker # via env vars, so the configured backend is used regardless of which # deployment initializes the ServerState actor first. -# mode: memory | file | redis +# mode: memory | redis # memory: requires an initialized Ray runtime (the launcher handles this # automatically; standalone scripts must call ray.init() first) -# file_path: required for `file` mode # redis_url / key_prefix: required for `redis` mode persistence: mode: redis diff --git a/cookbook/client/server/transformer/server_config.yaml b/cookbook/client/server/transformer/server_config.yaml index b5d8497f..7c8050c7 100644 --- a/cookbook/client/server/transformer/server_config.yaml +++ b/cookbook/client/server/transformer/server_config.yaml @@ -18,14 +18,12 @@ telemetry: # Top-level placement makes the launcher propagate this to every Ray worker # via env vars, so the configured backend is used regardless of which # deployment initializes the ServerState actor first. -# mode: memory | file | redis +# mode: memory | redis # memory: requires an initialized Ray runtime (the launcher handles this # automatically; standalone scripts must call ray.init() first) -# file_path: required for `file` mode # redis_url / key_prefix: required for `redis` mode persistence: - mode: file - file_path: /tmp/twinkle_state.json + mode: memory # Applications: each entry defines a service component deployed on the server applications: diff --git a/cookbook/client/tinker/upload_to_hub.py b/cookbook/client/tinker/upload_to_hub.py index da39cc02..b527ca3c 100644 --- a/cookbook/client/tinker/upload_to_hub.py +++ b/cookbook/client/tinker/upload_to_hub.py @@ -6,9 +6,11 @@ # # How it works: # 1. The server submits the upload as a background task and returns a -# request_id immediately, so the HTTP call never times out. -# 2. The client polls /upload_status/{request_id} every few seconds and -# blocks until the upload completes or raises on failure. +# Task_Envelope with a request_id immediately, so the HTTP call never times out. +# 2. The client's future layer long-polls /twinkle/retrieve_future and blocks +# until the upload reaches a terminal state, raising on failure. +# (`upload_to_hub` keeps its `poll_interval` / `async_upload` arguments for +# signature compatibility; both are deprecated and have no effect.) # # Prerequisites: # - Server must be running (see server.py / server_config.yaml) diff --git a/cookbook/client/twinkle/upload_to_hub.py b/cookbook/client/twinkle/upload_to_hub.py index 2c780e25..f2303622 100644 --- a/cookbook/client/twinkle/upload_to_hub.py +++ b/cookbook/client/twinkle/upload_to_hub.py @@ -6,9 +6,11 @@ # # How it works: # 1. The server submits the upload as a background task and returns a -# request_id immediately, so the HTTP call never times out. -# 2. The client polls /upload_status/{request_id} every few seconds and -# blocks until the upload completes or raises on failure. +# Task_Envelope with a request_id immediately, so the HTTP call never times out. +# 2. The client's future layer long-polls /twinkle/retrieve_future and blocks +# until the upload reaches a terminal state, raising on failure. +# (`upload_to_hub` keeps its `poll_interval` / `async_upload` arguments for +# signature compatibility; both are deprecated and have no effect.) # # Prerequisites: # - Server must be running (see server.py / server_config.yaml) diff --git a/docs/source_en/Usage Guide/Server and Client/Server.md b/docs/source_en/Usage Guide/Server and Client/Server.md index 5e67e37c..771e5624 100644 --- a/docs/source_en/Usage Guide/Server and Client/Server.md +++ b/docs/source_en/Usage Guide/Server and Client/Server.md @@ -183,10 +183,10 @@ telemetry: otlp_endpoint: http://localhost:4317 # Persistence: storage backend for ServerState (sessions, models, futures, etc.) -# mode: memory | file | redis +# mode: memory | redis persistence: - mode: file - file_path: /tmp/twinkle_state.json + mode: redis + redis_url: redis://localhost:6379/0 # Application list: Each entry defines a service component deployed on the Server applications: @@ -350,7 +350,7 @@ The difference from the Megatron backend is only in the `backend` parameter of t | `proxy_location` | HTTP proxy location (`EveryNode` or `HeadOnly`) | | `http_options` | HTTP listener config (`host`, `port`) | | `telemetry` | Observability config (`enabled`, `otlp_endpoint`) | -| `persistence` | State persistence config (`mode`, `file_path`, `redis_url`) | +| `persistence` | State persistence config (`mode`, `redis_url`) | | `applications` | Application component list | > The config file uses strict validation (`extra='forbid'`). Any misspelled field name will be rejected before startup. Use `twinkle-server check-config -c xxx.yaml` to detect errors early. @@ -418,8 +418,7 @@ Storage backend for ServerState (sessions, models, futures, etc.). | Field | Type | Default | Description | |-------|------|---------|-------------| -| `mode` | str | `memory` | `memory` / `file` / `redis` | -| `file_path` | str | — | Required for `file` mode, JSON file path | +| `mode` | str | `memory` | `memory` / `redis` | | `redis_url` | str | — | Required for `redis` mode, e.g. `redis://localhost:6379` | | `key_prefix` | str | `""` | Optional global key prefix | diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" index db71e41a..02467834 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/\346\234\215\345\212\241\347\253\257.md" @@ -183,10 +183,10 @@ telemetry: otlp_endpoint: http://localhost:4317 # 持久化：ServerState 的存储后端（sessions、models、futures 等） -# mode: memory | file | redis +# mode: memory | redis persistence: - mode: file - file_path: /tmp/twinkle_state.json + mode: redis + redis_url: redis://localhost:6379/0 # 应用列表：每个条目定义一个部署在 Server 上的服务组件 applications: @@ -350,7 +350,7 @@ Transformers 后端与 Megatron 后端的区别仅在 Model 服务的 `backend` | `proxy_location` | HTTP 代理位置（`EveryNode` 或 `HeadOnly`） | | `http_options` | HTTP 监听配置（`host`、`port`） | | `telemetry` | 可观测性配置（`enabled`、`otlp_endpoint`） | -| `persistence` | 状态持久化配置（`mode`、`file_path`、`redis_url`） | +| `persistence` | 状态持久化配置（`mode`、`redis_url`） | | `applications` | 应用组件列表 | > 配置文件启用了严格校验（`extra='forbid'`），任何拼写错误的字段名都会在启动前报错。可使用 `twinkle-server check-config -c xxx.yaml` 提前检测。 @@ -418,8 +418,7 @@ ServerState（sessions、models、futures 等）的存储后端。 | 字段 | 类型 | 默认值 | 说明 | |------|------|--------|------| -| `mode` | str | `memory` | `memory` / `file` / `redis` | -| `file_path` | str | — | `file` 模式必填，JSON 文件路径 | +| `mode` | str | `memory` | `memory` / `redis` | | `redis_url` | str | — | `redis` 模式必填，如 `redis://localhost:6379` | | `key_prefix` | str | `""` | 可选的全局 key 前缀 | diff --git a/src/twinkle/server/config/persistence.py b/src/twinkle/server/config/persistence.py index c61c18c1..cfb77192 100644 --- a/src/twinkle/server/config/persistence.py +++ b/src/twinkle/server/config/persistence.py @@ -11,7 +11,6 @@ # ServerState actor first. PERSISTENCE_ENV_KEYS: tuple[str, ...] = ( 'TWINKLE_PERSISTENCE_MODE', - 'TWINKLE_PERSISTENCE_FILE_PATH', 'TWINKLE_PERSISTENCE_REDIS_URL', 'TWINKLE_PERSISTENCE_KEY_PREFIX', ) @@ -22,16 +21,13 @@ class PersistenceConfig(BaseModel): model_config = ConfigDict(extra='forbid') - mode: Literal['memory', 'file', 'redis'] = 'memory' - file_path: str | None = None # required for file mode + mode: Literal['memory', 'redis'] = 'memory' redis_url: str | None = None # required for redis mode key_prefix: str = '' # optional global key prefix def to_env_vars(self) -> dict[str, str]: """Serialize this config to env var key/value pairs for worker propagation.""" env: dict[str, str] = {'TWINKLE_PERSISTENCE_MODE': self.mode} - if self.file_path: - env['TWINKLE_PERSISTENCE_FILE_PATH'] = self.file_path if self.redis_url: env['TWINKLE_PERSISTENCE_REDIS_URL'] = self.redis_url if self.key_prefix: @@ -50,7 +46,6 @@ def from_env(cls) -> PersistenceConfig | None: return None return cls( mode=mode, - file_path=os.environ.get('TWINKLE_PERSISTENCE_FILE_PATH'), redis_url=os.environ.get('TWINKLE_PERSISTENCE_REDIS_URL'), key_prefix=os.environ.get('TWINKLE_PERSISTENCE_KEY_PREFIX', ''), ) diff --git a/src/twinkle/server/deployment.py b/src/twinkle/server/deployment.py index ccd70096..cc67ae0e 100644 --- a/src/twinkle/server/deployment.py +++ b/src/twinkle/server/deployment.py @@ -35,10 +35,12 @@ from ray import serve from typing import Any +from twinkle.server.exceptions import TwinkleServerError from twinkle.server.telemetry.middleware import create_metrics_middleware from twinkle.server.telemetry.tracing import create_tracing_middleware from twinkle.server.utils.validation import verify_request_token from twinkle.utils.logger import get_logger +from twinkle_client.types.errors import ErrorPayload logger = get_logger() @@ -47,6 +49,24 @@ OnShutdown = Callable[[Any], Awaitable[None]] +async def twinkle_server_error_handler(request: Request, exc: TwinkleServerError) -> JSONResponse: + """Map a TwinkleServerError to a structured response, fields at the top level. + + Status code is the exception's ``error_code``; the body is an ``ErrorPayload`` + (``error`` / ``category`` / ``error_code`` / ``request_id``) placed at the top + level rather than nested under ``detail``. A Decision_Boundary-left rejection + (``category=user``) carries no traceback. + """ + request_id = getattr(request.state, 'request_id', None) or '' + payload = ErrorPayload( + error=(str(exc) or exc.__class__.__name__), + category=exc.category, + error_code=exc.error_code, + request_id=request_id, + ) + return JSONResponse(status_code=exc.error_code, content=payload.model_dump(mode='json', exclude_none=True)) + + def get_servable() -> Any: """The single definition of the servable-object accessor used by every builder. @@ -123,6 +143,8 @@ async def lifespan(app: FastAPI): app = FastAPI(lifespan=lifespan, **(fastapi_kwargs or {})) + app.add_exception_handler(TwinkleServerError, twinkle_server_error_handler) + # Registration order matters: FastAPI runs middleware LIFO, so the LAST # registered wraps the outermost layer. Register cleanup (if any) first so # it stays innermost, then the exception boundary, auth, tracing, metrics, diff --git a/src/twinkle/server/exceptions.py b/src/twinkle/server/exceptions.py index dc58ad81..1a3ba656 100644 --- a/src/twinkle/server/exceptions.py +++ b/src/twinkle/server/exceptions.py @@ -1,16 +1,46 @@ -"""Twinkle Server unified exception hierarchy.""" +"""Twinkle Server unified exception hierarchy. + +Every exception carries an ``error_code`` (an HTTP-status-shaped int in 400-599) and +a ``category`` (:class:`ErrorCategory`). A single ``TwinkleServerError`` exception +handler (see the gateway/model/sampler apps) reads these two attributes to build a +structured response whose fields sit at the top level of the body -- not nested +under ``detail``. +""" from __future__ import annotations +from twinkle_client.types.errors import ErrorCategory + class TwinkleServerError(Exception): - """Base class for all Twinkle Server exceptions.""" - pass + """Base class for all Twinkle Server exceptions. + + ``error_code`` / ``category`` are class-level defaults a subclass overrides; an + instance may also override them via keyword to avoid a subclass per status code. + """ + + error_code: int = 500 + category: ErrorCategory = ErrorCategory.Server + + def __init__( + self, + message: str = '', + *, + error_code: int | None = None, + category: ErrorCategory | None = None, + ) -> None: + super().__init__(message) + if error_code is not None: + self.error_code = error_code + if category is not None: + self.category = category class StateBackendError(TwinkleServerError): """State backend operation failed (connection lost, timeout, data serialization error, etc.).""" - pass + + error_code = 500 + category = ErrorCategory.Server class ConfigError(TwinkleServerError): @@ -22,6 +52,9 @@ class ConfigError(TwinkleServerError): re-running the server. """ + error_code = 500 + category = ErrorCategory.Server + def __init__( self, field: str, @@ -45,21 +78,72 @@ class ConfigParseError(TwinkleServerError): value violates a field/cross-field rule) and from ``FileNotFoundError`` (which signals that the source could not be read at all). """ - pass + + error_code = 500 + category = ErrorCategory.Server class ResourceExhaustedError(TwinkleServerError): """Resource exhausted — queue full, insufficient memory, connection pool exhausted, etc.""" - pass + error_code = 503 + category = ErrorCategory.Server + + +class RequestRejectedError(TwinkleServerError): + """Decision_Boundary-left failure: rejectable from the request body, deployment + config, and loaded schema alone, so it is returned with a real HTTP status code + and writes NO future record. + + Named ``RequestRejectedError`` rather than ``RequestValidationError`` to avoid a + collision with ``fastapi.exceptions.RequestValidationError``. The default is a + 400/User rejection; the subclasses below pin the specific status codes from the + Decision_Boundary placement table. + """ + + error_code = 400 + category = ErrorCategory.User + + +class TrainModeMismatchError(RequestRejectedError): + """The request's train mode does not match the deployment's (LoRA vs full).""" + + error_code = 400 + category = ErrorCategory.User + + +class InputTokensExceededError(RequestRejectedError): + """The request's input token count exceeds ``max_input_tokens``.""" + + error_code = 422 + category = ErrorCategory.User -class FullModeBusyError(TwinkleServerError): + +class BatchSizeError(RequestRejectedError): + """Batch size is incompatible with the data world size (too small / not a multiple).""" + + error_code = 422 + category = ErrorCategory.User + + +class RateLimitExceededError(RequestRejectedError): + """The request or token rate exceeds the configured limit.""" + + error_code = 429 + category = ErrorCategory.User + + +class FullModeBusyError(RequestRejectedError): """A full-parameter (exclusive) model deployment already has a holder. Full-parameter training rewrites the shared base-model weights, so a single - deployment can only host one training task at a time. + deployment can only host one training task at a time. It is a request rejection + (a second tenant is turned away), hence a 409 on the Decision_Boundary left. """ + error_code = 409 + category = ErrorCategory.User + def __init__(self, current_holder: str) -> None: self.current_holder = current_holder super().__init__('This deployment runs in full-parameter (exclusive) mode and is already ' diff --git a/src/twinkle/server/gateway/tinker_handlers.py b/src/twinkle/server/gateway/tinker_handlers.py index af2e8adb..268cd316 100644 --- a/src/twinkle/server/gateway/tinker_handlers.py +++ b/src/twinkle/server/gateway/tinker_handlers.py @@ -8,7 +8,6 @@ from __future__ import annotations import asyncio -import os from collections.abc import Callable from fastapi import Depends, FastAPI, HTTPException, Request, Response from tinker import types @@ -19,8 +18,8 @@ from twinkle.hub import HubOperation from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager +from twinkle.server.lifecycle.poll_config import long_poll_window, retrieve_poll_interval from twinkle.server.utils.task_errors import error_payload_from_stored -from twinkle.server.utils.task_queue import QueueState from twinkle.server.utils.validation import get_token_from_request from twinkle.utils.logger import get_logger @@ -83,8 +82,8 @@ async def retrieve_future(request: Request, self: GatewayServer = Depends(self_fn)) -> Any: """Retrieve the result of an async task with long polling.""" request_id = body.request_id - max_wait = float(os.environ.get('TWINKLE_LONG_POLL_TIMEOUT', '30')) - poll_interval = float(os.environ.get('TWINKLE_POLL_INTERVAL', '0.5')) + max_wait = long_poll_window() + poll_interval = retrieve_poll_interval() start = asyncio.get_running_loop().time() while True: @@ -92,7 +91,7 @@ async def retrieve_future(request: Request, if record is not None: status = record.get('status') - if status not in ('pending', 'queued', 'running', 'rate_limited'): + if status not in ('pending', 'queued', 'running'): break # ``record is None`` here means the future hasn't been written yet @@ -112,13 +111,6 @@ async def retrieve_future(request: Request, status = record.get('status') - if status == 'rate_limited': - return { - 'type': 'try_again', - 'queue_state': QueueState.PAUSED_RATE_LIMIT.value, - 'queue_state_reason': record.get('reason', 'Rate limit exceeded') - } - if status == 'failed': payload = error_payload_from_stored(record.get('result'), request_id=request_id) return payload.model_dump(mode='json', exclude_none=True) diff --git a/src/twinkle/server/gateway/twinkle_handlers.py b/src/twinkle/server/gateway/twinkle_handlers.py index c3de4d8a..396fb74c 100644 --- a/src/twinkle/server/gateway/twinkle_handlers.py +++ b/src/twinkle/server/gateway/twinkle_handlers.py @@ -6,6 +6,8 @@ """ from __future__ import annotations +import asyncio +import time from collections.abc import Callable from fastapi import Depends, FastAPI, HTTPException, Request from typing import TYPE_CHECKING @@ -15,8 +17,11 @@ import twinkle_client.types as types from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager, validate_user_path +from twinkle.server.lifecycle.envelope import envelope_from_record +from twinkle.server.lifecycle.poll_config import long_poll_window, retrieve_poll_interval from twinkle.server.utils.validation import get_token_from_request from twinkle.utils.logger import get_logger +from twinkle_client.types.lifecycle import TERMINAL_STATUSES logger = get_logger() @@ -101,6 +106,62 @@ async def session_heartbeat( raise HTTPException(status_code=404, detail='Unknown session') return types.SessionHeartbeatResponse() + @app.post('/twinkle/retrieve_future', response_model=types.TaskEnvelope) + async def retrieve_future( + request: Request, + body: types.RetrieveFutureRequest, + self: GatewayServer = Depends(self_fn), + ) -> types.TaskEnvelope: + """Long-poll a twinkle-native task to a terminal state. + + Returns 200 for every outcome except a request_id that stayed invisible for + a whole window -- the HTTP call succeeded, it successfully reported the + task's state. Unlike the tinker endpoint next door, ``completed`` with a + null result is a valid success (step / zero_grad / lr_step all return None), + so this handler never raises the tinker endpoint's + ``HTTPException(500, 'Task completed but no result found')``. + + A fixed interval, not exponential backoff: measured on real hardware, a + 0.05->1.0s doubling schedule is ~22% SLOWER per step because its interval + grows fastest across the 0.5-1.2s band where data-plane tasks actually + finish. See ``poll_config`` for the numbers. + """ + request_id = body.request_id + deadline = time.monotonic() + long_poll_window() + interval = retrieve_poll_interval() + record = None + + while True: + record = await self.state.get_future(request_id) + if record is not None and record.get('status') in TERMINAL_STATUSES: + return envelope_from_record(request_id, record) + if time.monotonic() >= deadline: + break + await asyncio.sleep(interval) + + if record is None: + # A whole window with no record: this endpoint knows how long it + # waited, which is why the missing-record decision lives here rather + # than in the caller. Cross-replica visibility lag is folded into the + # wait loop above instead of short-circuiting to 404. + raise HTTPException(status_code=404, detail=f'request_id {request_id} not found or expired') + return envelope_from_record(request_id, record) + + @app.post('/twinkle/cancel', response_model=types.CancelResponse) + async def cancel_future( + request: Request, + body: types.CancelRequest, + self: GatewayServer = Depends(self_fn), + ) -> types.CancelResponse: + """Best-effort cancel of a not-yet-started task. + + Drops the task from the compute queue only if it has not begun running; a + running or already-terminal task is reported but never interrupted, so cancel + can never corrupt in-flight GPU/optimizer state. + """ + result = await self.state.cancel_future(body.request_id) + return types.CancelResponse(**result) + @app.get('/twinkle/training_runs', response_model=types.TrainingRunsResponse) async def get_training_runs(request: Request, limit: int = 20, offset: int = 0) -> types.TrainingRunsResponse: token = get_token_from_request(request) diff --git a/src/twinkle/server/lifecycle/__init__.py b/src/twinkle/server/lifecycle/__init__.py new file mode 100644 index 00000000..b77b0c75 --- /dev/null +++ b/src/twinkle/server/lifecycle/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Server-side request-lifecycle package (server-request-lifecycle spec). + +Holds the pieces shared by the Submit_Endpoint and Retrieve_Endpoint: the single +FutureRecord -> TaskEnvelope mapping point, the poll/window configuration, and the +Submit_Endpoint shell. +""" diff --git a/src/twinkle/server/lifecycle/envelope.py b/src/twinkle/server/lifecycle/envelope.py new file mode 100644 index 00000000..cd5b2404 --- /dev/null +++ b/src/twinkle/server/lifecycle/envelope.py @@ -0,0 +1,64 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""The one place a FutureRecord becomes a TaskEnvelope. + +Both the Submit_Endpoint and the Retrieve_Endpoint go through this function so a +``failed`` status always lands in ``error`` and never in ``result``. Duplicating +this mapping per endpoint is how a task that failed inside the Inline_Fast_Path +window loses its payload. +""" +from __future__ import annotations + +from typing import Any + +from twinkle.server.utils.task_errors import error_payload_from_stored +from twinkle_client.types.lifecycle import TaskEnvelope + + +def envelope_from_record( + request_id: str, + record: dict[str, Any] | None, + *, + fallback_status: str = 'pending', +) -> TaskEnvelope: + """Map a stored ``FutureRecord`` dict to the wire ``TaskEnvelope``. + + Two behaviours are load-bearing: + + - The stored ``FutureRecord`` keeps a failure payload in its ``result`` field + (changing that would break state backward-compatibility). The wire split of + ``result`` / ``error`` is done here, which is why there must be exactly one + mapping point. + - A failure payload is reconstructed through Part 1's + ``error_payload_from_stored`` rather than a strict ``ErrorPayload.model_validate``. + Pre-spec records carry only ``{error, category}``; strict validation would + make retrieve return 500 for a record that should be a 200 + payload during + any rolling upgrade. + + ``completed`` with ``result is None`` is a valid success (``step`` / + ``zero_grad`` / ``lr_step`` all return ``None``); it is NOT treated as a + failure. The tinker endpoint's ``HTTPException(500, 'Task completed but no + result found')`` is a bug that this function deliberately does not copy. + + ``failed`` and ``cancelled`` both carry an ``ErrorPayload`` in ``error`` (the + cancel payload is stored the same way a failure payload is), so the client can + distinguish them by ``status`` while reading one field. + """ + record = record or {} + status = record.get('status', fallback_status) + common = dict( + queue_state=record.get('queue_state'), + queue_state_reason=record.get('queue_state_reason'), + ) + if status in ('failed', 'cancelled'): + return TaskEnvelope( + request_id=request_id, + status=status, + error=error_payload_from_stored(record.get('result'), request_id=request_id), + **common, + ) + return TaskEnvelope( + request_id=request_id, + status=status, + result=record.get('result') if status == 'completed' else None, + **common, + ) diff --git a/src/twinkle/server/lifecycle/poll_config.py b/src/twinkle/server/lifecycle/poll_config.py new file mode 100644 index 00000000..6aa6c47f --- /dev/null +++ b/src/twinkle/server/lifecycle/poll_config.py @@ -0,0 +1,79 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""The single declaration point for Long_Poll_Window and the retrieve poll interval. + +Both retrieve endpoints -- twinkle's ``POST /twinkle/retrieve_future`` and tinker's +``POST /retrieve_future`` -- read their timing from here so there is one source of truth, +and both long-poll at the same fixed interval. + +Why fixed and not exponential backoff (measured, and it overturned the original design): +the spec's argument for backoff was "a 5ms zero_grad that missed the Inline_Fast_Path +window should not wait out a full 500ms tick". On real hardware that case does not exist +-- control-plane ops measure 0.00-0.09s and are absorbed by the 50ms inline window, so +they never reach this endpoint at all. What does reach it is ``forward_backward`` at +0.52-0.65s, and there a 0.05->1.0s doubling schedule checks at 0.05/0.10/0.20/0.40/0.80, +landing on 0.80 for the whole cluster, whereas a fixed 0.5s checks at 0.05/0.55/1.05 and +catches most of it at 0.55. Backoff measured ~22% SLOWER per step (0.980s vs 0.800s mean) +because its interval grows fastest exactly across the band where real tasks finish. + +A denser ceiling (~0.2s) would beat both, at 2.5x the poll rate against a shared state +backend. That is a tuning knob, not a correctness one, and it is not worth optimising for +small-model step times: at production scale a data-plane call runs for minutes and any of +these granularities is noise. +""" +from __future__ import annotations + +import os + +from twinkle.utils.logger import get_logger + +logger = get_logger() + +# Documented assumption for a typical ingress / L7 gateway idle-connection limit. +# It is NOT hard-coded into any decision logic -- it is only the threshold at which +# ``long_poll_window()`` warns that a configured window is likely to be cut off. +# Operators can override the real limit for their deployment. +_ASSUMED_GATEWAY_IDLE_LIMIT = 60.0 + +# Default Long_Poll_Window. 30 < assumed gateway limit (60) and 30 < client HTTP +# timeout (90), so a retrieve request that waits a full window survives the gateway. +_DEFAULT_LONG_POLL_TIMEOUT = 30.0 + +# Fixed poll interval for BOTH retrieve endpoints (env ``TWINKLE_POLL_INTERVAL``, default +# 0.5s). Declared here so neither endpoint reads ``os.environ`` on its own. See the module +# docstring for the measurement that rejected exponential backoff. +_DEFAULT_POLL_INTERVAL = 0.5 + +# The last window value we warned about. ``long_poll_window()`` is on the hot path of both +# retrieve endpoints -- not just startup -- so an unguarded warning would fire on every +# retrieve request (roughly twice a second during training) to say something that only +# needs saying once. Keyed on the value rather than a bare bool so that a *changed* +# misconfiguration warns again instead of being swallowed by the first one. +_warned_window: float | None = None + + +def long_poll_window() -> float: + """Return the Long_Poll_Window in seconds (env ``TWINKLE_LONG_POLL_TIMEOUT``). + + Warns **once per configured value** when that value is at least the assumed gateway + idle limit: the retrieve endpoint is itself served through the gateway, so a window + past the gateway's limit would recreate the connection-cut problem this spec removes. + + The env var is re-read on every call (callers may change it, and tests do), so the + warning needs its own de-duplication -- this function runs per retrieve request, not + only at startup. + """ + global _warned_window + value = float(os.environ.get('TWINKLE_LONG_POLL_TIMEOUT', str(_DEFAULT_LONG_POLL_TIMEOUT))) + if value >= _ASSUMED_GATEWAY_IDLE_LIMIT and value != _warned_window: + _warned_window = value + logger.warning( + '[poll_config] TWINKLE_LONG_POLL_TIMEOUT=%.1fs is >= the assumed gateway idle limit ' + '(%.1fs). The retrieve endpoint is served through the gateway too, so a window this ' + 'large may be cut off mid-request. Lower it or confirm your gateway idle limit.', value, + _ASSUMED_GATEWAY_IDLE_LIMIT) + return value + + +def retrieve_poll_interval() -> float: + """Fixed poll interval shared by both retrieve endpoints (env ``TWINKLE_POLL_INTERVAL``).""" + return float(os.environ.get('TWINKLE_POLL_INTERVAL', str(_DEFAULT_POLL_INTERVAL))) diff --git a/src/twinkle/server/lifecycle/submit.py b/src/twinkle/server/lifecycle/submit.py new file mode 100644 index 00000000..005a2a3e --- /dev/null +++ b/src/twinkle/server/lifecycle/submit.py @@ -0,0 +1,150 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Submit_Endpoint shell and the named seams every queued handler shares. + +The Inline_Fast_Path wait itself (``submit_and_peek``) lives on +:class:`~twinkle.server.utils.task_queue.mixin.TaskQueueMixin`, since it operates on +queue state; this module owns the request-shaped pieces around it. +""" +from __future__ import annotations + +import uuid +from collections.abc import Callable, Coroutine +from fastapi import Request +from typing import Any + +from twinkle.data_format import InputFeature, Trajectory +from twinkle.server.utils.validation import get_session_id_from_request +from twinkle_client.types.lifecycle import TaskEnvelope + +# --------------------------------------------------------------------------- # +# Named seams. This spec implements the current semantics; the server-request-schema +# spec later replaces these function bodies without touching the shell or the return +# path, so the two specs edit disjoint regions. +# --------------------------------------------------------------------------- # + + +def to_backend_inputs(inputs: Any) -> Any: + """Seam A: convert raw dict/list inputs to InputFeature / Trajectory objects.""" + if isinstance(inputs, list) and inputs: + first = inputs[0] + if isinstance(first, dict) and 'input_ids' in first: + return [InputFeature(**item) for item in inputs] + return [Trajectory(**item) for item in inputs] + if isinstance(inputs, dict): + if 'input_ids' in inputs: + return [InputFeature(**inputs)] + return [Trajectory(**inputs)] + return inputs + + +def backend_kwargs(body: Any) -> dict[str, Any]: + """Seam B: the passthrough kwargs forwarded to the backend call.""" + return body.model_extra or {} + + +def input_metrics(self, body: Any, *, data_parallel: bool = False) -> dict[str, Any]: + """Seam C: scheduling metrics (input_tokens, and batch_size/data_world_size). + + Defensive shape (isinstance guards + .get defaults) because the body is not yet + strictly validated; a non-dict element must not raise here. + """ + inputs = body.inputs + inputs_list = inputs if isinstance(inputs, list) else [inputs] + input_tokens = sum(len(inp.get('input_ids', [])) if isinstance(inp, dict) else 0 for inp in inputs_list) + metrics: dict[str, Any] = {'input_tokens': input_tokens} + if data_parallel: + metrics['batch_size'] = len(inputs_list) + metrics['data_world_size'] = self.data_world_size + return metrics + + +def resolve_twinkle_adapter_name(request: Request, adapter_name: str | None) -> str | None: + """Build a stable per-session adapter name, falling back to request_id for older clients.""" + if adapter_name is None or adapter_name == '': + return None + owner_id = get_session_id_from_request(request) or request.state.request_id + return owner_id + '-' + adapter_name + + +async def run_submit( + self, + request: Request, + body: Any, + *, + task_type: str, + backend_call: Callable[..., Coroutine], + metrics: Callable[[Any, Any], dict[str, Any]] | None = None, + assert_resource: bool = True, +) -> TaskEnvelope: + """The common Submit_Endpoint judgment sequence, called by every queued + twinkle-native handler instead of being repeated in each. + + Order is load-bearing: request start -> adapter resolution -> ``submit_and_peek`` + (whose ``schedule_task`` runs preflight). Every admission check runs before any + state write, so a rejected request writes nothing. + + A plain helper, not a signature-rewriting decorator: each handler keeps its natural + FastAPI signature so the app stays shallow enough for Ray Serve to cloudpickle (a + signature-patching wrapper once deepened the route graph past CPython's C-stack + recursion guard during ``serve.ingress``). + + Four queued endpoints deliberately do NOT route through here and call + ``submit_and_peek`` / ``submit_background_and_peek`` directly: model + ``add_adapter_to_model`` (creates the adapter, so the resource assertion cannot + apply and it owns the train_mode/full-mode checks), model ``upload_to_hub`` (pure + I/O, background task), and sampler ``sample`` / ``sample_to_data_plane`` (no adapter + semantics). Zero-write and admission still hold for them because both live inside + ``schedule_task`` -> ``_perform_preflight_checks``, not in this shell. + + ``backend_call(self, body, adapter_name, token)`` runs the endpoint-specific call + and returns the JSON-safe task result. ``metrics(self, body)`` supplies scheduling + kwargs; omit it for control-plane ops. ``assert_resource`` guards on the adapter + existing before the work runs; set it False for endpoints that create/drop it. + """ + token = await self._on_request_start(request) + adapter_name = resolve_twinkle_adapter_name(request, body.adapter_name) + + schedule_kwargs = metrics(self, body) if metrics is not None else {} + + async def _task(): + if assert_resource: + self.assert_resource_exists(adapter_name) + return await backend_call(self, body, adapter_name, token) + + # ---- Idempotent dedup: a client-supplied seq_id makes a retried stateful op + # apply at most once. Claim (session_id, adapter, seq_id) -> request_id atomically + # before enqueue; a hit returns the original task's envelope instead of re-enqueuing. + # Only grad-mutating client calls set seq_id, so other endpoints skip this. + # + # The adapter must be part of the key. Each client model object owns its own seq + # counter starting at 1, while session_id is process-global -- so two adapters + # trained from one process would collide on (session, seq) and the second + # forward_backward would be dropped as a duplicate AND handed the first adapter's + # loss. That is a silent wrong-result bug, i.e. the exact failure this dedup + # exists to prevent, one level up. ---- + request_id = f'req_{uuid.uuid4().hex}' + seq_id = getattr(body, 'seq_id', None) + dedup_key = None + if seq_id is not None: + session_id = get_session_id_from_request(request) or request.state.request_id + dedup_key = f'seq::{session_id}::{adapter_name or "-"}::{seq_id}' + ttl = int(self._task_queue_config.effective_execution_timeout) + 60 + prior_request_id = await self.state.claim_seq(dedup_key, request_id, ttl) + if prior_request_id is not None: + return await self._peek_terminal(prior_request_id, fallback_status='pending') + + # ---- Decision_Boundary: preflight (in schedule_task) then peek ---- + try: + return await self.submit_and_peek( + _task, model_id=adapter_name, token=token, task_type=task_type, request_id=request_id, **schedule_kwargs) + except Exception: + # Release the seq claim only when the task never made it onto the queue -- + # decided by whether a future record exists, NOT by the exception type. A + # preflight rejection raises before any record is written, so releasing lets + # a retry re-enqueue. But a failure *after* enqueue (e.g. a transient state + # error inside the peek) leaves a live task that will still run; releasing + # there would let a retry enqueue a duplicate -> the exact double-apply this + # dedup prevents. When unsure (record exists), keep the claim. + if dedup_key is not None and await self.state.get_future(request_id) is None: + await self.state.release_seq(dedup_key) + raise diff --git a/src/twinkle/server/model/app.py b/src/twinkle/server/model/app.py index 0b73cbc6..77873b6d 100644 --- a/src/twinkle/server/model/app.py +++ b/src/twinkle/server/model/app.py @@ -216,7 +216,7 @@ def mark_unhealthy(self) -> None: self._model_unhealthy = True async def _probe_after_timeout(self) -> None: - """Fired by ComputeWorker on a backend timeout: probe and log liveness (R3#2).""" + """Fired by ComputeWorker on a backend timeout: probe and log liveness.""" result = await self.check_model_health() logger.warning('[Model] post-timeout liveness probe: %s', result) diff --git a/src/twinkle/server/model/twinkle_handlers.py b/src/twinkle/server/model/twinkle_handlers.py index c743c5d3..ce0c724e 100644 --- a/src/twinkle/server/model/twinkle_handlers.py +++ b/src/twinkle/server/model/twinkle_handlers.py @@ -2,27 +2,29 @@ """ Twinkle-native model handler mixin. -All endpoints are prefixed /twinkle/... and use schedule_task_and_wait() returning -results directly (synchronous from the client's perspective). -self_fn is injected via FastAPI Depends to obtain the ModelManagement instance at request time. +All queued endpoints are prefixed /twinkle/... and return a Task_Envelope via the +shared ``run_submit`` judgment sequence: the handler submits work and returns +immediately, and the client's Client_Future_Layer resolves the envelope to a +terminal state. self_fn is injected via FastAPI Depends to obtain the +ModelManagement instance at request time. """ from __future__ import annotations import torch -import traceback from collections.abc import Callable from fastapi import Depends, FastAPI, HTTPException, Request from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING if TYPE_CHECKING: from .app import ModelManagement import twinkle_client.types as types -from twinkle.data_format import InputFeature, Trajectory from twinkle.server.checkpoint import (_resolve_client_save_dir, create_checkpoint_manager, create_training_run_manager, validate_user_path) -from twinkle.server.exceptions import FullModeBusyError +from twinkle.server.exceptions import RequestRejectedError, TrainModeMismatchError +from twinkle.server.lifecycle.submit import (backend_kwargs, input_metrics, resolve_twinkle_adapter_name, run_submit, + to_backend_inputs) from twinkle.server.model.utils import (data_plane_request_shape, merge_forward_kwargs, resolve_data_plane_model_inputs, select_output_rows) from twinkle.server.utils.validation import get_session_id_from_request @@ -31,28 +33,20 @@ logger = get_logger() -def _parse_inputs(inputs: Any): - """Convert raw dict/list inputs to InputFeature or Trajectory objects.""" - if isinstance(inputs, list) and inputs: - first = inputs[0] - if isinstance(first, dict) and 'input_ids' in first: - return [InputFeature(**item) for item in inputs] - else: - return [Trajectory(**item) for item in inputs] - elif isinstance(inputs, dict): - if 'input_ids' in inputs: - return [InputFeature(**inputs)] - else: - return [Trajectory(**inputs)] - return inputs +def _dp_metrics(self, body): + """Scheduling metrics for inline data-parallel endpoints (forward / forward_backward).""" + return input_metrics(self, body, data_parallel=True) -def _get_twinkle_adapter_name(request: Request, adapter_name: str | None) -> str | None: - """Build a stable per-session adapter name, falling back to request_id for older clients.""" - if adapter_name is None or adapter_name == '': - return None - owner_id = get_session_id_from_request(request) or request.state.request_id - return owner_id + '-' + adapter_name +def _tokens_only_metrics(self, body): + """Scheduling metrics for inline non-data-parallel endpoints (forward_only).""" + return input_metrics(self, body, data_parallel=False) + + +def _data_plane_metrics(self, body): + """Scheduling metrics derived from DataRef shape for *_from_data_plane endpoints.""" + input_tokens, batch_size = data_plane_request_shape(body) + return {'input_tokens': input_tokens, 'batch_size': batch_size, 'data_world_size': self.data_world_size} def _register_twinkle_routes(app: FastAPI, self_fn: Callable[[], ModelManagement]) -> None: @@ -74,460 +68,362 @@ async def model_healthz( return JSONResponse(status_code=503, content=result) return result - async def run_task(coro): - """Await a schedule_task_and_wait coroutine and surface any exception as a - structured HTTP 500 response so the client receives the full traceback instead - of an opaque connection-level error. - - Note: HTTPException is re-raised directly to preserve its status code and detail. - """ - try: - return await coro - except HTTPException: - raise # Re-raise HTTPException directly to preserve status code - except Exception: - logger.error(traceback.format_exc()) - raise HTTPException(status_code=500, detail=traceback.format_exc()) - @app.post('/twinkle/create', response_model=types.CreateResponse) async def create(request: Request, body: types.CreateRequest, self: ModelManagement = Depends(self_fn)) -> types.CreateResponse: await self._on_request_start(request) return types.CreateResponse() - @app.post('/twinkle/forward', response_model=types.ForwardResponse) + # ------------------------------------------------------------------ # + # Inline data / forward family + # ------------------------------------------------------------------ # + + @app.post('/twinkle/forward', response_model=types.TaskEnvelope) async def forward(request: Request, body: types.ForwardRequest, - self: ModelManagement = Depends(self_fn)) -> types.ForwardResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} - inputs = _parse_inputs(body.inputs) + async def _call(self, body, adapter_name, token): ret = await self.call_backend( self.model.forward, - inputs=inputs, + inputs=to_backend_inputs(body.inputs), adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) + **backend_kwargs(body)) return {'result': ret} - inputs_list = body.inputs if isinstance(body.inputs, list) else [body.inputs] - input_tokens = sum(len(inp.get('input_ids', [])) if isinstance(inp, dict) else 0 for inp in inputs_list) - batch_size = len(inputs_list) - return await run_task( - self.schedule_task_and_wait( - _task, - model_id=adapter_name, - token=token, - input_tokens=input_tokens, - batch_size=batch_size, - data_world_size=self.data_world_size, - task_type='forward', - )) + return await run_submit(self, request, body, task_type='forward', backend_call=_call, metrics=_dp_metrics) - @app.post('/twinkle/forward_from_data_plane', response_model=types.ForwardResponse) - async def forward_from_data_plane( - request: Request, - body: types.DataPlaneForwardRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.ForwardResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + @app.post('/twinkle/forward_only', response_model=types.TaskEnvelope) + async def forward_only( + request: Request, body: types.ForwardOnlyRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) - kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) + async def _call(self, body, adapter_name, token): ret = await self.call_backend( - self.model.forward, - inputs=_parse_inputs(raw_inputs), - adapter_name=adapter_name, - **kwargs, - ) + self.model.forward_only, + inputs=to_backend_inputs(body.inputs), + adapter_name=self.resolve_model_adapter_name(adapter_name), + **backend_kwargs(body)) return {'result': ret} - input_tokens, batch_size = data_plane_request_shape(body) - return await run_task( - self.schedule_task_and_wait( - _task, - model_id=adapter_name, - token=token, - input_tokens=input_tokens, - batch_size=batch_size, - data_world_size=self.data_world_size, - task_type='forward_from_data_plane', - )) + return await run_submit( + self, request, body, task_type='forward_only', backend_call=_call, metrics=_tokens_only_metrics) - @app.post('/twinkle/remove_adapter') - async def remove_adapter( - request: Request, - body: types.AdapterRequest, - self: ModelManagement = Depends(self_fn), - ) -> dict[str, str]: - """Release a drained tenant's in-memory training adapter.""" - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) - - async def _task(): - await self._cleanup_adapter(adapter_name) - return {'status': 'ok'} + @app.post('/twinkle/forward_backward', response_model=types.TaskEnvelope) + async def forward_backward( + request: Request, body: types.ForwardRequest, self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - return await run_task( - self.schedule_task_and_wait( - _task, - model_id=adapter_name, - token=token, - task_type='remove_adapter', - )) + async def _call(self, body, adapter_name, token): - @app.post('/twinkle/forward_only', response_model=types.ForwardResponse) - async def forward_only( - request: Request, - body: types.ForwardOnlyRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.ForwardResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + def first_element(data): + while isinstance(data, list): + if len(data) == 0: + return None + data = data[0] + return data - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} - inputs = _parse_inputs(body.inputs) + all_inputs = to_backend_inputs(body.inputs) + for inputs in all_inputs: + for key in inputs: + if isinstance(inputs[key], list) and isinstance(first_element(inputs[key]), (int, float)): + inputs[key] = torch.tensor(inputs[key]) ret = await self.call_backend( - self.model.forward_only, - inputs=inputs, + self.model.forward_backward, + inputs=all_inputs, adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) - return {'result': ret} - - inputs_list = body.inputs if isinstance(body.inputs, list) else [body.inputs] - input_tokens = sum(len(inp.get('input_ids', [])) if isinstance(inp, dict) else 0 for inp in inputs_list) - return await run_task( - self.schedule_task_and_wait( - _task, - model_id=adapter_name, - token=token, - input_tokens=input_tokens, - task_type='forward_only', - )) - - @app.post('/twinkle/forward_only_from_data_plane', response_model=types.ForwardResponse) - async def forward_only_from_data_plane( - request: Request, - body: types.DataPlaneForwardOnlyRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.ForwardResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) - - async def _task(): - self.assert_resource_exists(adapter_name) - raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) - inputs = _parse_inputs(raw_inputs) - kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) - ret = await self.call_backend(self.model.forward_only, inputs=inputs, adapter_name=adapter_name, **kwargs) - if body.output_ref is not None: - rows = select_output_rows( - ret, - batch_size=len(inputs), - output_fields=body.output_fields, - ) - output_ref = await self.data_plane.append(body.output_ref, rows) - return {'result': output_ref.model_dump()} + **backend_kwargs(body)) return {'result': ret} - input_tokens, batch_size = data_plane_request_shape(body) - return await run_task( - self.schedule_task_and_wait( - _task, - model_id=adapter_name, - token=token, - input_tokens=input_tokens, - batch_size=batch_size, - data_world_size=self.data_world_size, - task_type='forward_only_from_data_plane', - )) + return await run_submit( + self, request, body, task_type='forward_backward', backend_call=_call, metrics=_dp_metrics) - @app.post('/twinkle/calculate_loss', response_model=types.CalculateLossResponse) + @app.post('/twinkle/calculate_loss', response_model=types.TaskEnvelope) async def calculate_loss( - request: Request, - body: types.AdapterRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.CalculateLossResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + request: Request, body: types.AdapterRequest, self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): ret = await self.call_backend( - self.model.calculate_loss, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + self.model.calculate_loss, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **backend_kwargs(body)) return {'result': ret} - return await run_task( - self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='calculate_loss')) + return await run_submit(self, request, body, task_type='calculate_loss', backend_call=_call) - @app.post('/twinkle/backward') - async def backward(request: Request, body: types.AdapterRequest, self: ModelManagement = Depends(self_fn)) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + @app.post('/twinkle/backward', response_model=types.TaskEnvelope) + async def backward(request: Request, body: types.AdapterRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): await self.call_backend( - self.model.backward, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + self.model.backward, adapter_name=self.resolve_model_adapter_name(adapter_name), **backend_kwargs(body)) - await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='backward')) + return await run_submit(self, request, body, task_type='backward', backend_call=_call) - @app.post('/twinkle/forward_backward', response_model=types.ForwardBackwardResponse) - async def forward_backward( - request: Request, - body: types.ForwardRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.ForwardBackwardResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + # ------------------------------------------------------------------ # + # Data-plane forward family (DataRef inputs; response only enters the contract) + # ------------------------------------------------------------------ # - def first_element(data): - while isinstance(data, list): - if len(data) == 0: - return None - data = data[0] - return data + @app.post('/twinkle/forward_from_data_plane', response_model=types.TaskEnvelope) + async def forward_from_data_plane( + request: Request, body: types.DataPlaneForwardRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} - all_inputs = _parse_inputs(body.inputs) - for inputs in all_inputs: - for key in inputs: - if isinstance(inputs[key], list) and isinstance(first_element(inputs[key]), (int, float)): - inputs[key] = torch.tensor(inputs[key]) + async def _call(self, body, adapter_name, token): + raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) + kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) ret = await self.call_backend( - self.model.forward_backward, - inputs=all_inputs, - adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) + self.model.forward, inputs=to_backend_inputs(raw_inputs), adapter_name=adapter_name, **kwargs) return {'result': ret} - inputs_list = body.inputs if isinstance(body.inputs, list) else [body.inputs] - input_tokens = sum(len(inp.get('input_ids', [])) if isinstance(inp, dict) else 0 for inp in inputs_list) - batch_size = len(inputs_list) - return await run_task( - self.schedule_task_and_wait( - _task, - model_id=adapter_name, - token=token, - input_tokens=input_tokens, - batch_size=batch_size, - data_world_size=self.data_world_size, - task_type='forward_backward', - )) + return await run_submit( + self, request, body, task_type='forward_from_data_plane', backend_call=_call, metrics=_data_plane_metrics) + + @app.post('/twinkle/forward_only_from_data_plane', response_model=types.TaskEnvelope) + async def forward_only_from_data_plane( + request: Request, body: types.DataPlaneForwardOnlyRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: + + async def _call(self, body, adapter_name, token): + raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) + inputs = to_backend_inputs(raw_inputs) + kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) + ret = await self.call_backend(self.model.forward_only, inputs=inputs, adapter_name=adapter_name, **kwargs) + if body.output_ref is not None: + rows = select_output_rows(ret, batch_size=len(inputs), output_fields=body.output_fields) + output_ref = await self.data_plane.append(body.output_ref, rows) + return {'result': output_ref.model_dump()} + return {'result': ret} + + return await run_submit( + self, + request, + body, + task_type='forward_only_from_data_plane', + backend_call=_call, + metrics=_data_plane_metrics) - @app.post('/twinkle/forward_backward_from_data_plane', response_model=types.ForwardBackwardResponse) + @app.post('/twinkle/forward_backward_from_data_plane', response_model=types.TaskEnvelope) async def forward_backward_from_data_plane( - request: Request, - body: types.DataPlaneForwardRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.ForwardBackwardResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + request: Request, body: types.DataPlaneForwardRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) + async def _call(self, body, adapter_name, token): raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) ret = await self.call_backend( - self.model.forward_backward, - inputs=_parse_inputs(raw_inputs), - adapter_name=adapter_name, - **kwargs, - ) + self.model.forward_backward, inputs=to_backend_inputs(raw_inputs), adapter_name=adapter_name, **kwargs) return {'result': ret} - input_tokens, batch_size = data_plane_request_shape(body) - return await run_task( - self.schedule_task_and_wait( - _task, - model_id=adapter_name, - token=token, - input_tokens=input_tokens, - batch_size=batch_size, - data_world_size=self.data_world_size, - task_type='forward_backward_from_data_plane', - )) + return await run_submit( + self, + request, + body, + task_type='forward_backward_from_data_plane', + backend_call=_call, + metrics=_data_plane_metrics) + + # ------------------------------------------------------------------ # + # Optimizer / control plane + # ------------------------------------------------------------------ # - @app.post('/twinkle/clip_grad_norm', response_model=types.ClipGradNormResponse) + @app.post('/twinkle/clip_grad_norm', response_model=types.TaskEnvelope) async def clip_grad_norm( - request: Request, - body: types.AdapterRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.ClipGradNormResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + request: Request, body: types.AdapterRequest, self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): ret = await self.call_backend( - self.model.clip_grad_norm, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + self.model.clip_grad_norm, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **backend_kwargs(body)) return {'result': str(ret)} - return await run_task( - self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='clip_grad_norm')) + return await run_submit(self, request, body, task_type='clip_grad_norm', backend_call=_call) - @app.post('/twinkle/step') - async def step(request: Request, body: types.AdapterRequest, self: ModelManagement = Depends(self_fn)) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + @app.post('/twinkle/step', response_model=types.TaskEnvelope) + async def step(request: Request, body: types.AdapterRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): await self.call_backend( - self.model.step, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + self.model.step, adapter_name=self.resolve_model_adapter_name(adapter_name), **backend_kwargs(body)) - await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='step')) + return await run_submit(self, request, body, task_type='step', backend_call=_call) - @app.post('/twinkle/zero_grad') - async def zero_grad(request: Request, body: types.AdapterRequest, self: ModelManagement = Depends(self_fn)) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + @app.post('/twinkle/zero_grad', response_model=types.TaskEnvelope) + async def zero_grad(request: Request, body: types.AdapterRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): await self.call_backend( - self.model.zero_grad, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + self.model.zero_grad, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **backend_kwargs(body)) - await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='zero_grad')) + return await run_submit(self, request, body, task_type='zero_grad', backend_call=_call) - @app.post('/twinkle/lr_step') - async def lr_step(request: Request, body: types.AdapterRequest, self: ModelManagement = Depends(self_fn)) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + @app.post('/twinkle/lr_step', response_model=types.TaskEnvelope) + async def lr_step(request: Request, body: types.AdapterRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): await self.call_backend( - self.model.lr_step, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) + self.model.lr_step, adapter_name=self.resolve_model_adapter_name(adapter_name), **backend_kwargs(body)) - await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='lr_step')) + return await run_submit(self, request, body, task_type='lr_step', backend_call=_call) - @app.post('/twinkle/clip_grad_and_step') + @app.post('/twinkle/clip_grad_and_step', response_model=types.TaskEnvelope) async def clip_grad_and_step( - request: Request, - body: types.ClipGradAndStepRequest, - self: ModelManagement = Depends(self_fn), - ) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + request: Request, body: types.ClipGradAndStepRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): await self.call_backend( self.model.clip_grad_and_step, max_grad_norm=body.max_grad_norm, norm_type=body.norm_type, adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs, - ) + **backend_kwargs(body)) - await run_task( - self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='clip_grad_and_step')) + return await run_submit(self, request, body, task_type='clip_grad_and_step', backend_call=_call) - @app.post('/twinkle/get_train_configs', response_model=types.GetTrainConfigsResponse) + @app.post('/twinkle/get_train_configs', response_model=types.TaskEnvelope) async def get_train_configs( - request: Request, - body: types.AdapterRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.GetTrainConfigsResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + request: Request, body: types.AdapterRequest, self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): ret = await self.call_backend( self.model.get_train_configs, adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) + **backend_kwargs(body)) return {'result': ret} - return await run_task( - self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='get_train_configs')) + return await run_submit(self, request, body, task_type='get_train_configs', backend_call=_call) - @app.post('/twinkle/set_loss') - async def set_loss(request: Request, body: types.SetLossRequest, self: ModelManagement = Depends(self_fn)) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + @app.post('/twinkle/set_loss', response_model=types.TaskEnvelope) + async def set_loss(request: Request, body: types.SetLossRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): await self.call_backend( self.model.set_loss, body.loss_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) + **backend_kwargs(body)) - await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_loss')) + return await run_submit(self, request, body, task_type='set_loss', backend_call=_call) - @app.post('/twinkle/set_optimizer') + @app.post('/twinkle/set_optimizer', response_model=types.TaskEnvelope) async def set_optimizer( - request: Request, - body: types.SetOptimizerRequest, - self: ModelManagement = Depends(self_fn), - ) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + request: Request, body: types.SetOptimizerRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): await self.call_backend( self.model.set_optimizer, body.optimizer_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) + **backend_kwargs(body)) - await run_task( - self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_optimizer')) + return await run_submit(self, request, body, task_type='set_optimizer', backend_call=_call) - @app.post('/twinkle/set_lr_scheduler') + @app.post('/twinkle/set_lr_scheduler', response_model=types.TaskEnvelope) async def set_lr_scheduler( - request: Request, - body: types.SetLrSchedulerRequest, - self: ModelManagement = Depends(self_fn), - ) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + request: Request, body: types.SetLrSchedulerRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): await self.call_backend( self.model.set_lr_scheduler, body.scheduler_cls, adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) + **backend_kwargs(body)) + + return await run_submit(self, request, body, task_type='set_lr_scheduler', backend_call=_call) + + @app.post('/twinkle/set_template', response_model=types.TaskEnvelope) + async def set_template( + request: Request, body: types.SetTemplateRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: + + async def _call(self, body, adapter_name, token): + await self.call_backend( + self.model.set_template, + body.template_cls, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **backend_kwargs(body)) + + return await run_submit(self, request, body, task_type='set_template', backend_call=_call) + + @app.post('/twinkle/set_processor', response_model=types.TaskEnvelope) + async def set_processor( + request: Request, body: types.SetProcessorRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: + + async def _call(self, body, adapter_name, token): + await self.call_backend( + self.model.set_processor, + body.processor_cls, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **backend_kwargs(body)) + + return await run_submit(self, request, body, task_type='set_processor', backend_call=_call) + + @app.post('/twinkle/add_metric', response_model=types.TaskEnvelope) + async def add_metric(request: Request, body: types.AddMetricRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: + + async def _call(self, body, adapter_name, token): + from twinkle_client.common.serialize import deserialize_object + metric_cls = deserialize_object(body.metric_cls) + await self.call_backend( + self.model.add_metric, + metric_cls, + is_training=body.is_training, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **backend_kwargs(body)) + + return await run_submit(self, request, body, task_type='add_metric', backend_call=_call) + + @app.post('/twinkle/apply_patch', response_model=types.TaskEnvelope) + async def apply_patch( + request: Request, body: types.ApplyPatchRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: + + async def _call(self, body, adapter_name, token): + from twinkle_client.common.serialize import deserialize_object + patch_cls = deserialize_object(body.patch_cls) + await self.call_backend( + self.model.apply_patch, + patch_cls, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **backend_kwargs(body)) - await run_task( - self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_lr_scheduler')) + return await run_submit(self, request, body, task_type='apply_patch', backend_call=_call) - @app.post('/twinkle/save', response_model=types.SaveResponse) + @app.post('/twinkle/calculate_metric', response_model=types.TaskEnvelope) + async def calculate_metric( + request: Request, body: types.CalculateMetricRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: + + async def _call(self, body, adapter_name, token): + ret = await self.call_backend( + self.model.calculate_metric, + is_training=body.is_training, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **backend_kwargs(body)) + return {'result': ret} + + return await run_submit(self, request, body, task_type='calculate_metric', backend_call=_call) + + # ------------------------------------------------------------------ # + # Checkpoint I/O (need the caller token) + # ------------------------------------------------------------------ # + + @app.post('/twinkle/save', response_model=types.TaskEnvelope) async def save(request: Request, body: types.SaveRequest, - self: ModelManagement = Depends(self_fn)) -> types.SaveResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') checkpoint_name = checkpoint_manager.get_ckpt_name(body.name) save_dir = checkpoint_manager.get_save_dir(model_id=adapter_name, is_sampler=body.is_sampler) @@ -542,19 +438,16 @@ async def _task(): output_dir=save_dir, adapter_name=self.resolve_model_adapter_name(adapter_name), save_optimizer=body.save_optimizer, - **extra_kwargs) + **backend_kwargs(body)) return {'twinkle_path': twinkle_path, 'checkpoint_dir': checkpoint_dir} - return await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='save')) + return await run_submit(self, request, body, task_type='save', backend_call=_call) - @app.post('/twinkle/load') - async def load(request: Request, body: types.LoadRequest, self: ModelManagement = Depends(self_fn)) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + @app.post('/twinkle/load', response_model=types.TaskEnvelope) + async def load(request: Request, body: types.LoadRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + async def _call(self, body, adapter_name, token): checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') resolved = checkpoint_manager.resolve_load_path(body.name) await self.call_backend( @@ -564,21 +457,16 @@ async def _task(): adapter_name=self.resolve_model_adapter_name(adapter_name), load_optimizer=body.load_optimizer, token=token, - **extra_kwargs) + **backend_kwargs(body)) - await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='load')) + return await run_submit(self, request, body, task_type='load', backend_call=_call) - @app.post('/twinkle/resume_from_checkpoint', response_model=types.TrainingProgressResponse) + @app.post('/twinkle/resume_from_checkpoint', response_model=types.TaskEnvelope) async def resume_from_checkpoint( - request: Request, - body: types.ResumeFromCheckpointRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.TrainingProgressResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + request: Request, body: types.ResumeFromCheckpointRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: - async def _task(): - self.assert_resource_exists(adapter_name) + async def _call(self, body, adapter_name, token): checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') resolved = checkpoint_manager.resolve_load_path(body.name) checkpoint_dir = ( @@ -588,107 +476,71 @@ async def _task(): self.model.resume_from_checkpoint, checkpoint_dir, resume_only_model=body.resume_only_model, - adapter_name=self.resolve_model_adapter_name(adapter_name), - ) + adapter_name=self.resolve_model_adapter_name(adapter_name)) return {'result': ret} - return await run_task(self.schedule_task_and_wait(_task, task_type='resume')) + return await run_submit(self, request, body, task_type='resume', backend_call=_call) - @app.post('/twinkle/upload_to_hub', response_model=types.UploadToHubResponse) - async def upload_to_hub( - request: Request, - body: types.UploadToHubRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.UploadToHubResponse: - token = await self._on_request_start(request) + # ------------------------------------------------------------------ # + # Adapter lifecycle (create / drop the adapter itself: no resource assert) + # ------------------------------------------------------------------ # - async def _task(): - if body.checkpoint_dir.startswith('twinkle://'): - checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') - parsed = checkpoint_manager.parse_twinkle_path(body.checkpoint_dir) - if not parsed: - raise ValueError(f'Invalid twinkle path format: {body.checkpoint_dir}') - checkpoint_id = parsed.checkpoint_id - model_id_to_load = parsed.training_run_id - checkpoint = checkpoint_manager.get(model_id_to_load, checkpoint_id) - if not checkpoint: - raise ValueError(f'Checkpoint not found or access denied: {body.checkpoint_dir}') - checkpoint_dir = str( - checkpoint_manager.get_ckpt_dir(model_id=model_id_to_load, checkpoint_id=checkpoint_id)) - else: - checkpoint_dir = body.checkpoint_dir - await self.call_backend( - self.model.upload_to_hub, - checkpoint_dir=checkpoint_dir, - hub_model_id=body.hub_model_id, - hub_token=body.hub_token or token, - async_upload=False, - ) + @app.post('/twinkle/remove_adapter', response_model=types.TaskEnvelope) + async def remove_adapter( + request: Request, body: types.AdapterRequest, self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: + """Release a drained tenant's in-memory training adapter.""" - future_ref = await self.schedule_background_task(_task, task_type='upload_to_hub') - request_id = future_ref.get('request_id') - if request_id is None: - raise HTTPException(status_code=500, detail=f'Upload task scheduling failed: {future_ref}') - return types.UploadToHubResponse(request_id=request_id) + async def _call(self, body, adapter_name, token): + await self._cleanup_adapter(adapter_name) + return {'status': 'ok'} - @app.get('/twinkle/upload_status/{request_id}', response_model=types.UploadStatusResponse) - async def upload_status( - request: Request, - request_id: str, - self: ModelManagement = Depends(self_fn), - ) -> types.UploadStatusResponse: - await self._on_request_start(request) - record = await self.state.get_future(request_id) - if record is None: - raise HTTPException(status_code=404, detail=f'Upload task not found: {request_id}') - status = record.get('status', 'unknown') - error = None - if status == 'failed': - error = record.get('result', {}).get('error', 'Unknown error') - return types.UploadStatusResponse(request_id=request_id, status=status, error=error) - - @app.post('/twinkle/add_adapter_to_model', response_model=types.AddAdapterResponse) + return await run_submit( + self, request, body, task_type='remove_adapter', backend_call=_call, assert_resource=False) + + @app.post('/twinkle/add_adapter_to_model', response_model=types.TaskEnvelope) async def add_adapter_to_model( request: Request, body: types.AddAdapterRequest, self: ModelManagement = Depends(self_fn), - ) -> types.AddAdapterResponse: - assert body.adapter_name, 'You need to specify a valid `adapter_name`' + ) -> types.TaskEnvelope: + # This endpoint creates the adapter, so it cannot use the standard resource + # assertion. The Decision_Boundary left checks (train_mode 400 / full-mode + # 409) run here, before any state write, raising RequestRejectedError + # subclasses (zero future writes). + # + # Raised, not asserted: a missing adapter_name is decidable from the request body + # alone, so it owes the caller a real 400. A bare `assert` would surface as a 500 + # ('the server broke') and would vanish entirely under `python -O`, letting an + # empty adapter_name through to the backend. + if not body.adapter_name: + raise RequestRejectedError('`adapter_name` is required and must be non-empty.') token = await self._on_request_start(request) if not validate_user_path(token, body.adapter_name): - raise HTTPException(status_code=400, detail=f'Invalid adapter_name: {body.adapter_name}') - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + raise RequestRejectedError(f'Invalid adapter_name: {body.adapter_name}') + adapter_name = resolve_twinkle_adapter_name(request, body.adapter_name) session_id = get_session_id_from_request(request) try: resolved_save_dir = _resolve_client_save_dir(body.save_dir).as_posix() if body.save_dir else None except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) + raise RequestRejectedError(str(exc)) from exc + + from twinkle_client.common.serialize import deserialize_object + config = deserialize_object(body.config) + + # ---- Decision_Boundary left: validate against the deployment's train_mode ---- + if self.is_full_mode and config is not None: + raise TrainModeMismatchError('This deployment runs in full-parameter (exclusive) mode; pass ' + 'config=None (do not send a LoraConfig).') + if (not self.is_full_mode) and config is None: + raise TrainModeMismatchError('This deployment runs in LoRA mode; a LoraConfig is required.') + if self.is_full_mode: + # Raises FullModeBusyError (409) if another tenant holds the exclusive deployment. + self.assert_full_mode_available(adapter_name) async def _task(): from peft import LoraConfig - - from twinkle_client.common.serialize import deserialize_object - config = deserialize_object(body.config) extra_kwargs = body.model_extra or {} training_run_manager = create_training_run_manager(token, client_type='twinkle') - - # Validate the supplied config against the deployment's train_mode. - if self.is_full_mode and config is not None: - raise HTTPException( - status_code=400, - detail='This deployment runs in full-parameter (exclusive) mode; pass config=None ' - '(do not send a LoraConfig).') - if (not self.is_full_mode) and config is None: - raise HTTPException( - status_code=400, detail='This deployment runs in LoRA mode; a LoraConfig is required.') - - # In full mode ensure the exclusive deployment is free before touching state. - if self.is_full_mode: - try: - self.assert_full_mode_available(adapter_name) - except FullModeBusyError as e: - raise HTTPException(status_code=409, detail=str(e)) - lora_config = None if isinstance(config, LoraConfig): lora_config = types.LoraConfig(rank=config.r, train_unembed=False, train_mlp=True, train_attn=True) @@ -718,132 +570,40 @@ async def _task(): training_run_manager.save(adapter_name, run_config) return {'status': 'ok', 'adapter_name': adapter_name} - return await run_task( - self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='add_adapter_to_model')) - - @app.post('/twinkle/apply_patch') - async def apply_patch( - request: Request, - body: types.ApplyPatchRequest, - self: ModelManagement = Depends(self_fn), - ) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) - - async def _task(): - from twinkle_client.common.serialize import deserialize_object - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} - patch_cls = deserialize_object(body.patch_cls) - await self.call_backend( - self.model.apply_patch, - patch_cls, - adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) - - await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='apply_patch')) + return await self.submit_and_peek(_task, model_id=adapter_name, token=token, task_type='add_adapter_to_model') - @app.post('/twinkle/add_metric') - async def add_metric( - request: Request, - body: types.AddMetricRequest, - self: ModelManagement = Depends(self_fn), - ) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) + # ------------------------------------------------------------------ # + # Hub upload (pure I/O -> background task; state-tracked via Retrieve_Endpoint) + # ------------------------------------------------------------------ # - async def _task(): - from twinkle_client.common.serialize import deserialize_object - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} - metric_cls = deserialize_object(body.metric_cls) - await self.call_backend( - self.model.add_metric, - metric_cls, - is_training=body.is_training, - adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) - - await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='add_metric')) - - @app.post('/twinkle/set_template') - async def set_template( - request: Request, - body: types.SetTemplateRequest, - self: ModelManagement = Depends(self_fn), - ) -> None: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) - - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} - await self.call_backend( - self.model.set_template, - body.template_cls, - adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) - - await run_task(self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_template')) - - @app.post('/twinkle/set_processor') - async def set_processor( + @app.post('/twinkle/upload_to_hub', response_model=types.TaskEnvelope) + async def upload_to_hub( request: Request, - body: types.SetProcessorRequest, + body: types.UploadToHubRequest, self: ModelManagement = Depends(self_fn), - ) -> None: + ) -> types.TaskEnvelope: token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} + if body.checkpoint_dir.startswith('twinkle://'): + checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') + parsed = checkpoint_manager.parse_twinkle_path(body.checkpoint_dir) + if not parsed: + raise ValueError(f'Invalid twinkle path format: {body.checkpoint_dir}') + checkpoint = checkpoint_manager.get(parsed.training_run_id, parsed.checkpoint_id) + if not checkpoint: + raise ValueError(f'Checkpoint not found or access denied: {body.checkpoint_dir}') + checkpoint_dir = str( + checkpoint_manager.get_ckpt_dir( + model_id=parsed.training_run_id, checkpoint_id=parsed.checkpoint_id)) + else: + checkpoint_dir = body.checkpoint_dir await self.call_backend( - self.model.set_processor, - body.processor_cls, - adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) - - await run_task( - self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='set_processor')) - - @app.post('/twinkle/calculate_metric', response_model=types.CalculateMetricResponse) - async def calculate_metric( - request: Request, - body: types.CalculateMetricRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.CalculateMetricResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) - - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} - ret = await self.call_backend( - self.model.calculate_metric, - is_training=body.is_training, - adapter_name=self.resolve_model_adapter_name(adapter_name), - **extra_kwargs) - return {'result': ret} - - return await run_task( - self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='calculate_metric')) - - @app.post('/twinkle/get_state_dict', response_model=types.GetStateDictResponse) - async def get_state_dict( - request: Request, - body: types.GetStateDictRequest, - self: ModelManagement = Depends(self_fn), - ) -> types.GetStateDictResponse: - token = await self._on_request_start(request) - adapter_name = _get_twinkle_adapter_name(request, body.adapter_name) - - async def _task(): - self.assert_resource_exists(adapter_name) - extra_kwargs = body.model_extra or {} - ret = await self.call_backend( - self.model.get_state_dict, adapter_name=self.resolve_model_adapter_name(adapter_name), **extra_kwargs) - return {'result': ret} + self.model.upload_to_hub, + checkpoint_dir=checkpoint_dir, + hub_model_id=body.hub_model_id, + hub_token=body.hub_token or token, + async_upload=False, + ) - return await run_task( - self.schedule_task_and_wait(_task, model_id=adapter_name, token=token, task_type='get_state_dict')) + return await self.submit_background_and_peek(_task, task_type='upload_to_hub') diff --git a/src/twinkle/server/sampler/twinkle_handlers.py b/src/twinkle/server/sampler/twinkle_handlers.py index c5428b05..ed58785d 100644 --- a/src/twinkle/server/sampler/twinkle_handlers.py +++ b/src/twinkle/server/sampler/twinkle_handlers.py @@ -22,10 +22,11 @@ import twinkle_client.types as types from twinkle.data_format import InputFeature, SamplingParams, Trajectory +from twinkle.server.exceptions import RequestRejectedError +from twinkle.server.lifecycle.submit import resolve_twinkle_adapter_name from twinkle.server.telemetry.correlation import MODEL_ID from twinkle.server.telemetry.tracing import traced_operation from twinkle.server.utils.task_errors import task_error_payload -from twinkle.server.utils.validation import get_session_id_from_request from twinkle.utils.logger import get_logger from twinkle_client.common.json_utils import json_safe @@ -51,11 +52,8 @@ def _serialize_input_feature(feature: dict) -> dict: def _get_twinkle_sampler_adapter_name(request: Request, adapter_name: str | None) -> str | None: - """Build a stable per-session adapter name, falling back to request_id for older clients.""" - if adapter_name is None or adapter_name == '': - return None - owner_id = get_session_id_from_request(request) or request.state.request_id - return owner_id + '-' + adapter_name + """Per-session adapter name; delegates to the shared lifecycle resolver.""" + return resolve_twinkle_adapter_name(request, adapter_name) def _build_rollout_rows_and_tags( @@ -215,30 +213,14 @@ def _register_twinkle_sampler_routes(app: FastAPI, self_fn: Callable[[], Sampler It is wired in via Depends so it is resolved lazily at request time. """ - async def run_task(coro): - """Await a schedule_task_and_wait coroutine and surface any exception as a - structured HTTP 500 response so the client receives the full traceback instead - of an opaque connection-level error. - - Note: HTTPException is re-raised directly to preserve its status code and detail. - """ - try: - return await coro - except HTTPException: - raise - except Exception: - logger.error(traceback.format_exc()) - raise HTTPException(status_code=500, detail=traceback.format_exc()) - @app.post('/twinkle/create', response_model=types.CreateResponse) async def create(request: Request, self: SamplerManagement = Depends(self_fn)) -> types.CreateResponse: """Health check / session creation endpoint.""" return types.CreateResponse() - @app.post('/twinkle/sample', response_model=types.SampleResponseModelList) - async def sample( - request: Request, body: types.SampleRequest, - self: SamplerManagement = Depends(self_fn)) -> types.SampleResponseModelList: + @app.post('/twinkle/sample', response_model=types.TaskEnvelope) + async def sample(request: Request, body: types.SampleRequest, + self: SamplerManagement = Depends(self_fn)) -> types.TaskEnvelope: """Sample completions from the model. Supports Trajectory or InputFeature inputs, with optional LoRA adapter. @@ -293,26 +275,21 @@ async def _task(): adapter_name=full_adapter_name, adapter_path=adapter_path, ) - return types.SampleResponseModelList(samples=_to_sample_response_models(responses)) + return types.SampleResponseModelList(samples=_to_sample_response_models(responses)).model_dump() # Calculate metrics for queue scheduling inputs_list = body.inputs if isinstance(body.inputs, list) else [body.inputs] input_tokens = sum(len(inp.get('input_ids', [])) if isinstance(inp, dict) else 0 for inp in inputs_list) - return await run_task( - self.schedule_task_and_wait( - _task, - token=token, - input_tokens=input_tokens, - task_type='sample', - )) + return await self.submit_and_peek(_task, token=token, input_tokens=input_tokens, task_type='sample') - @app.post('/twinkle/sample_to_data_plane', response_model=types.DataRef) + @app.post('/twinkle/sample_to_data_plane', response_model=types.TaskEnvelope) async def sample_to_data_plane( request: Request, body: types.DataPlaneSampleRequest, self: SamplerManagement = Depends(self_fn), - ) -> types.DataRef: - """Generate a complete group, store it server-side, and return its DataRef.""" + ) -> types.TaskEnvelope: + """Generate a complete group, store it server-side, and return a Task_Envelope + whose result is the stored group's DataRef.""" token = await self._on_request_start(request) if not self.data_plane.enabled: raise HTTPException(status_code=503, detail='sample_to_data_plane requires data_plane_url') @@ -341,7 +318,10 @@ async def sample_to_data_plane( params = SamplingParams.from_dict(params_dict) submission_id = uuid.uuid4().hex - async def _admit(): + async def _generate_and_store(): + # vLLM async engine owns generation concurrency, so the whole + # admit -> await -> store sequence runs as one background future + # (outside the serial compute queue) and its result is the DataRef. await self.call_backend( self.sampler.submit_generation, submission_id, @@ -350,33 +330,19 @@ async def _admit(): adapter_name=full_adapter_name, adapter_path=adapter_path, ) - return submission_id - - inline_inputs = body.inputs if isinstance(body.inputs, list) else [body.inputs] - input_tokens = ( - body.input_ref.num_tokens if body.input_ref is not None else sum( - len(item.get('input_ids', [])) for item in inline_inputs if isinstance(item, dict))) - await run_task( - self.schedule_task_and_wait( - _admit, - model_id=full_adapter_name or None, - token=token, - input_tokens=input_tokens, - task_type='sample_admission', - )) + responses = await _await_generation(self, submission_id, + self._task_queue_config.effective_execution_timeout) + rows, tags = _build_rollout_rows_and_tags( + _to_sample_response_models(responses), + group_ids=body.group_ids, + policy_version=body.policy_version, + adapter_uri=body.adapter_uri, + ) + ref = await self.data_plane.put([json_safe(item) for item in rows], kind='rollout', tags=tags) + return ref.model_dump() - responses = await _await_generation(self, submission_id, self._task_queue_config.effective_execution_timeout) - rows, tags = _build_rollout_rows_and_tags( - _to_sample_response_models(responses), - group_ids=body.group_ids, - policy_version=body.policy_version, - adapter_uri=body.adapter_uri, - ) - return await self.data_plane.put( - [json_safe(item) for item in rows], - kind='rollout', - tags=tags, - ) + return await self.submit_background_and_peek( + _generate_and_store, model_id=full_adapter_name or None, task_type='sample_to_data_plane') @app.post('/twinkle/unload_adapter_paths') async def unload_adapter_paths( @@ -417,7 +383,11 @@ async def add_adapter_to_sampler( self: SamplerManagement = Depends(self_fn), ) -> types.AddAdapterResponse: """Add a LoRA adapter to the sampler.""" - assert body.adapter_name, 'You need to specify a valid `adapter_name`' + # Raised, not asserted: decidable from the request body alone, so it owes the caller + # a real 400 rather than an AssertionError surfacing as a 500 -- and a bare assert + # would vanish under `python -O`, letting an empty adapter_name reach the backend. + if not body.adapter_name: + raise RequestRejectedError('`adapter_name` is required and must be non-empty.') full_adapter_name = _get_twinkle_sampler_adapter_name(request, body.adapter_name) from peft import LoraConfig diff --git a/src/twinkle/server/state/backend/__init__.py b/src/twinkle/server/state/backend/__init__.py index 6d46eda0..420bcef2 100644 --- a/src/twinkle/server/state/backend/__init__.py +++ b/src/twinkle/server/state/backend/__init__.py @@ -1,7 +1,6 @@ from twinkle.server.config.persistence import PersistenceConfig from .base import StateBackend from .factory import create_backend -from .file_backend import FileBackend from .redis_backend import RedisBackend # NOTE: ``RayActorBackend`` is intentionally NOT imported here. It top-level @@ -10,7 +9,6 @@ # from ``.memory_backend`` — as ``create_backend`` does lazily for memory mode. __all__ = [ 'StateBackend', - 'FileBackend', 'RedisBackend', 'PersistenceConfig', 'create_backend', diff --git a/src/twinkle/server/state/backend/factory.py b/src/twinkle/server/state/backend/factory.py index be24c782..842a67df 100644 --- a/src/twinkle/server/state/backend/factory.py +++ b/src/twinkle/server/state/backend/factory.py @@ -26,17 +26,10 @@ def create_backend(config: PersistenceConfig | None = None) -> StateBackend: match config.mode: case 'memory': - # Deferred import: RayActorBackend pulls in ``ray``, which is an - # optional dependency. Importing it lazily means callers that - # never select memory mode (e.g. file/redis users) do not need - # ray installed just to load this factory. + # Deferred import keeps the module-level import graph light; the Twinkle + # server always runs on Ray Serve, so ``ray`` is available here. from .memory_backend import RayActorBackend return RayActorBackend(key_prefix=config.key_prefix) - case 'file': - if not config.file_path: - raise ValueError('file_path is required for file persistence mode') - from .file_backend import FileBackend - return FileBackend(config.file_path) case 'redis': if not config.redis_url: raise ValueError('redis_url is required for redis persistence mode') diff --git a/src/twinkle/server/state/backend/file_backend.py b/src/twinkle/server/state/backend/file_backend.py deleted file mode 100644 index 42f94e43..00000000 --- a/src/twinkle/server/state/backend/file_backend.py +++ /dev/null @@ -1,235 +0,0 @@ -from __future__ import annotations - -import asyncio -import fcntl -import json -import os -import tempfile -import time -from collections.abc import Callable -from contextlib import contextmanager -from fnmatch import fnmatch -from typing import Any - -from .base import StateBackend - - -class FileBackend(StateBackend): - """Local JSON file-based persistent state backend. - - Storage format is a single JSON file: - ``{key: {"value": ..., "expire_at": float|null}}``. - File I/O is wrapped with ``asyncio.to_thread`` to avoid blocking the - event loop. Every operation that reads-or-writes goes through a sibling - ``.lock`` file held with ``fcntl.LOCK_EX`` so concurrent processes and - coroutines all serialize on the same critical section — that is the only - way ``update_atomic`` can give a meaningful atomicity guarantee against a - concurrent ``set`` / ``delete`` on the same key. - """ - - def __init__(self, file_path: str) -> None: - self._file_path = file_path - self._lock_path = f'{file_path}.lock' - self._init_file() - - def _init_file(self) -> None: - """Auto-create the data file, the lock file, and any missing parent dir.""" - dir_path = os.path.dirname(self._file_path) - if dir_path and not os.path.exists(dir_path): - os.makedirs(dir_path, exist_ok=True) - if not os.path.exists(self._file_path): - with open(self._file_path, 'w', encoding='utf-8') as f: - json.dump({}, f) - # Touch the lock file so flock has a stable inode across processes. - if not os.path.exists(self._lock_path): - with open(self._lock_path, 'a', encoding='utf-8'): - pass - - # ----- lock + file primitives ---------------------------------------- # - - @contextmanager - def _locked(self): - """Hold an exclusive flock on the sibling lock file for the block.""" - with open(self._lock_path, 'a+', encoding='utf-8') as lock_f: - fcntl.flock(lock_f.fileno(), fcntl.LOCK_EX) - try: - yield - finally: - fcntl.flock(lock_f.fileno(), fcntl.LOCK_UN) - - def _load_sync(self) -> dict[str, dict[str, Any]]: - try: - with open(self._file_path, encoding='utf-8') as f: - return json.load(f) - except (json.JSONDecodeError, FileNotFoundError): - return {} - - def _save_sync(self, data: dict[str, dict[str, Any]]) -> None: - """Write temp file then atomic-replace. Caller must hold ``_locked``.""" - # Drop expired entries on the write path so the file never grows - # unbounded with stale keys. - now = time.time() - data = {k: v for k, v in data.items() if v.get('expire_at') is None or v['expire_at'] > now} - - dir_path = os.path.dirname(self._file_path) or '.' - fd = tempfile.NamedTemporaryFile( - mode='w', - suffix='.tmp', - dir=dir_path, - delete=False, - encoding='utf-8', - ) - try: - json.dump(data, fd, ensure_ascii=False) - fd.flush() - os.fsync(fd.fileno()) - fd.close() - os.replace(fd.name, self._file_path) - except BaseException: - if os.path.exists(fd.name): - os.unlink(fd.name) - raise - - def _is_expired(self, entry: dict[str, Any]) -> bool: - expire_at = entry.get('expire_at') - return expire_at is not None and time.time() >= expire_at - - # ----- public API: every op runs under one lock ---------------------- # - - def _set_sync(self, key: str, value: Any, ttl: int | None) -> None: - with self._locked(): - data = self._load_sync() - expire_at = (time.time() + ttl) if ttl is not None else None - data[key] = {'value': value, 'expire_at': expire_at} - self._save_sync(data) - - async def set(self, key: str, value: Any, ttl: int | None = None) -> None: - await asyncio.to_thread(self._set_sync, key, value, ttl) - - def _get_sync(self, key: str) -> Any | None: - with self._locked(): - data = self._load_sync() - entry = data.get(key) - if entry is None: - return None - if self._is_expired(entry): - del data[key] - self._save_sync(data) - return None - return entry['value'] - - async def get(self, key: str) -> Any | None: - return await asyncio.to_thread(self._get_sync, key) - - def _delete_sync(self, key: str) -> None: - with self._locked(): - data = self._load_sync() - if key in data: - del data[key] - self._save_sync(data) - - async def delete(self, key: str) -> None: - await asyncio.to_thread(self._delete_sync, key) - - def _exists_sync(self, key: str) -> bool: - with self._locked(): - data = self._load_sync() - entry = data.get(key) - if entry is None: - return False - if self._is_expired(entry): - del data[key] - self._save_sync(data) - return False - return True - - async def exists(self, key: str) -> bool: - return await asyncio.to_thread(self._exists_sync, key) - - def _keys_sync(self, pattern: str) -> list[str]: - with self._locked(): - data = self._load_sync() - result: list[str] = [] - expired_keys: list[str] = [] - for key, entry in data.items(): - if self._is_expired(entry): - expired_keys.append(key) - continue - if fnmatch(key, pattern): - result.append(key) - if expired_keys: - for key in expired_keys: - del data[key] - self._save_sync(data) - return result - - async def keys(self, pattern: str) -> list[str]: - return await asyncio.to_thread(self._keys_sync, pattern) - - async def count(self, pattern: str) -> int: - return len(await self.keys(pattern)) - - def _set_nx_sync(self, key: str, value: Any, ttl: int | None) -> bool: - with self._locked(): - data = self._load_sync() - entry = data.get(key) - if entry is not None and not self._is_expired(entry): - return False - expire_at = (time.time() + ttl) if ttl is not None else None - data[key] = {'value': value, 'expire_at': expire_at} - self._save_sync(data) - return True - - async def set_nx(self, key: str, value: Any, ttl: int | None = None) -> bool: - return await asyncio.to_thread(self._set_nx_sync, key, value, ttl) - - def _update_atomic_sync( - self, - key: str, - transform: Callable[[Any | None], Any | None], - ttl: int | None, - ) -> Any | None: - with self._locked(): - data = self._load_sync() - entry = data.get(key) - current = None if (entry is None or self._is_expired(entry)) else entry['value'] - new_value = transform(current) - if new_value is None: - return current - expire_at = (time.time() + ttl) if ttl is not None else None - data[key] = {'value': new_value, 'expire_at': expire_at} - self._save_sync(data) - return new_value - - async def update_atomic( - self, - key: str, - transform: Callable[[Any | None], Any | None], - ttl: int | None = None, - ) -> Any | None: - return await asyncio.to_thread(self._update_atomic_sync, key, transform, ttl) - - def _mget_sync(self, keys: list[str]) -> list[Any | None]: - with self._locked(): - data = self._load_sync() - results: list[Any | None] = [] - for key in keys: - entry = data.get(key) - if entry is None or self._is_expired(entry): - results.append(None) - else: - results.append(entry['value']) - return results - - async def mget(self, keys: list[str]) -> list[Any | None]: - return await asyncio.to_thread(self._mget_sync, keys) - - async def close(self) -> None: - """File backend has no persistent connection — nothing to release.""" - pass - - async def health_check(self) -> bool: - try: - return os.access(self._file_path, os.W_OK) - except OSError: - return False diff --git a/src/twinkle/server/state/future_manager.py b/src/twinkle/server/state/future_manager.py index 3a9bd917..c12cc5e0 100644 --- a/src/twinkle/server/state/future_manager.py +++ b/src/twinkle/server/state/future_manager.py @@ -14,7 +14,7 @@ logger = get_logger() # Status sets used by the do-not-regress guard inside the atomic transform. -_TERMINAL_STATUSES = frozenset({'completed', 'failed'}) +_TERMINAL_STATUSES = frozenset({'completed', 'failed', 'cancelled'}) _NON_TERMINAL_STATUSES = frozenset({'pending', 'queued', 'running'}) @@ -80,6 +80,25 @@ def _future_record_transform( return updated +_CANCELLABLE_STATUSES = frozenset({'pending', 'queued'}) + + +def _cancel_if_not_started_transform(existing: dict | None, *, payload: dict, now: str) -> dict | None: + """Atomic transform: cancel iff the task has not started (pending/queued). + + Returns ``None`` (no change) for running/terminal/missing records so a task + already executing is never interrupted -- cancel is best-effort on the queue. + """ + status = existing.get('status') if existing is not None else None + if status not in _CANCELLABLE_STATUSES: + return None + updated = dict(existing) + updated['status'] = 'cancelled' + updated['result'] = payload + updated['updated_at'] = now + return updated + + class FutureManager(BaseManager[FutureRecord]): """Manage future state, terminal retention, and immutable task deadlines.""" @@ -130,6 +149,21 @@ async def store_status( ), ) + async def cancel_if_pending(self, request_id: str) -> str | None: + """Cancel a task iff it has not started; return the resulting status. + + Writes a terminal ``cancelled`` record (carrying a user ErrorPayload) only + when the current status is pending/queued -- a running task is left alone. + Returns the record's status after the attempt, or ``None`` if there is no + record for ``request_id``. + """ + payload = {'error': 'Task cancelled by client', 'category': 'user', 'error_code': 499} + result = await self._backend.update_atomic( + self._make_key(request_id), + functools.partial(_cancel_if_not_started_transform, payload=payload, now=_now_iso()), + ) + return result.get('status') if result else None + # ----- Cleanup ----- async def cleanup_expired( @@ -140,7 +174,7 @@ async def cleanup_expired( ) -> int: """Expire future records without ever deleting a non-terminal one. - Processing matrix (design §5.2): + Processing matrix: | status | replica alive | past deadline | action | |--------------|---------------|---------------|-------------------| diff --git a/src/twinkle/server/state/model_manager.py b/src/twinkle/server/state/model_manager.py index 58edf59b..e791c454 100644 --- a/src/twinkle/server/state/model_manager.py +++ b/src/twinkle/server/state/model_manager.py @@ -103,11 +103,11 @@ async def unregister_replica(self, replica_id: str) -> None: await self._replicas.unregister(replica_id) async def touch_replica_last_seen(self, replica_id: str) -> None: - """Refresh a replica's liveness timestamp (R4#6).""" + """Refresh a replica's liveness timestamp.""" await self._replicas.touch_last_seen(replica_id) async def get_alive_replica_ids(self, liveness_threshold: float) -> set[str]: - """Return replicas considered alive (R4#7, R4#8). + """Return replicas considered alive. A replica is alive when it has a ``last_seen`` within ``liveness_threshold``, OR when it has a ``max_loras`` entry but no ``last_seen`` yet (registered diff --git a/src/twinkle/server/state/server_state.py b/src/twinkle/server/state/server_state.py index fc8014a0..2317558d 100644 --- a/src/twinkle/server/state/server_state.py +++ b/src/twinkle/server/state/server_state.py @@ -280,6 +280,35 @@ async def get_future(self, request_id: str) -> dict[str, Any] | None: record = await self._future_mgr.get(request_id) return record.model_dump() if record is not None else None + async def claim_seq(self, dedup_key: str, request_id: str, ttl: int) -> str | None: + """Idempotency claim for a client seq_id. + + Atomically records ``dedup_key -> request_id`` if unseen and returns ``None`` + (caller proceeds to enqueue). If the key already exists, returns the prior + ``request_id`` so the caller can return that task's envelope instead of + enqueuing a duplicate. ``ttl`` bounds the dedup window. + """ + if await self._backend.set_nx(dedup_key, request_id, ttl=ttl): + return None + return await self._backend.get(dedup_key) + + async def release_seq(self, dedup_key: str) -> None: + """Drop a seq dedup claim (used when the claimed request never enqueued, e.g. + preflight rejected it) so a retry can be admitted rather than see a phantom.""" + await self._backend.delete(dedup_key) + + async def cancel_future(self, request_id: str) -> dict[str, Any]: + """Best-effort cancel: drop the task iff it has not started running. + + Returns ``{'cancelled': bool, 'state': str}`` where ``state`` is the task's + status after the attempt (``cancelled`` if just dropped or already cancelled, + ``running``/``completed``/``failed`` if too late, ``not_found`` if unknown). + """ + status = await self._future_mgr.cancel_if_pending(request_id) + if status is None: + return {'cancelled': False, 'state': 'not_found'} + return {'cancelled': status == 'cancelled', 'state': status} + async def store_future_status( self, request_id: str, @@ -300,13 +329,12 @@ async def store_future_status( - RUNNING: Task currently executing - COMPLETED: Task completed successfully (result required) - FAILED: Task failed with error (result contains error payload) - - RATE_LIMITED: Task rejected due to rate limiting (reason required) Args: request_id: Unique identifier for the request. - status: Task status string (pending/queued/running/completed/failed/rate_limited). + status: Task status string (pending/queued/running/completed/failed). model_id: Optional associated model_id. - reason: Optional reason string (used for rate_limited status). + reason: Optional reason string. result: Optional result data (used for completed/failed status). queue_state: Optional queue state for tinker client (active/paused_rate_limit/paused_capacity). queue_state_reason: Optional reason for the queue state. @@ -386,7 +414,7 @@ async def cleanup_expired_resources(self) -> dict[str, int]: } async def touch_replica_last_seen(self, replica_id: str) -> None: - """Refresh a replica's liveness timestamp in the shared registry (R4#6).""" + """Refresh a replica's liveness timestamp in the shared registry.""" await self._model_mgr.touch_replica_last_seen(replica_id) async def _cleanup_loop(self) -> None: diff --git a/src/twinkle/server/utils/task_errors.py b/src/twinkle/server/utils/task_errors.py index 031f71cf..cfa85257 100644 --- a/src/twinkle/server/utils/task_errors.py +++ b/src/twinkle/server/utils/task_errors.py @@ -2,7 +2,7 @@ """Construction and backward-compatible reading of failure payloads. ``ErrorPayload`` is the single representation of a failure both on the wire and in -state (R5). This module owns the two entry points that produce/repair it. +state. This module owns the two entry points that produce/repair it. """ from __future__ import annotations diff --git a/src/twinkle/server/utils/task_queue/config.py b/src/twinkle/server/utils/task_queue/config.py index 4de98e0d..71a9a8a0 100644 --- a/src/twinkle/server/utils/task_queue/config.py +++ b/src/twinkle/server/utils/task_queue/config.py @@ -32,6 +32,10 @@ class TaskQueueConfig(BaseModel): token_cleanup_multiplier: Multiplier for token cleanup threshold. token_cleanup_interval: How often to run cleanup task (seconds). max_input_tokens: Maximum allowed input tokens per request. + inline_fast_path_timeout: Upper bound (seconds) on how long submit briefly + polls the record so a millisecond-scale control-plane op (step / zero_grad) + completes in a single HTTP round trip instead of forcing a retrieve. + Must remain < Long_Poll_Window. """ model_config = ConfigDict(extra='forbid') @@ -45,6 +49,7 @@ class TaskQueueConfig(BaseModel): token_cleanup_multiplier: float = Field(default=10.0, ge=0) token_cleanup_interval: float = Field(default=60.0, ge=0) max_input_tokens: int = Field(default=16000, ge=1) + inline_fast_path_timeout: float = Field(default=0.05, gt=0) @property def effective_execution_timeout(self) -> float: diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/utils/task_queue/mixin.py index 83526c39..49037c78 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/utils/task_queue/mixin.py @@ -17,10 +17,13 @@ from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Any +from twinkle.server.exceptions import BatchSizeError, ConfigError, InputTokensExceededError, RateLimitExceededError +from twinkle.server.lifecycle.envelope import envelope_from_record +from twinkle.server.lifecycle.poll_config import long_poll_window from twinkle.server.telemetry.middleware import get_task_metrics from twinkle.server.utils.task_errors import task_error_payload from twinkle.utils.logger import get_logger -from twinkle_client.types.errors import ErrorCategory +from twinkle_client.types.lifecycle import TERMINAL_STATUSES, TaskEnvelope from .config import TaskQueueConfig from .rate_limiter import RateLimiter from .types import BackendBusyError, QueuedTask, QueueState, TaskStatus @@ -37,7 +40,7 @@ class TaskQueueMixin: Execution paths --------------- - 1. Compute queue (schedule_task / schedule_task_and_wait): + 1. Compute queue (schedule_task / submit_and_peek): Single background worker, serial execution, round-robin across queues. Use for GPU operations: forward, backward, step, save, load, etc. @@ -83,6 +86,18 @@ def _init_task_queue( logger.warning( '[TaskQueue] execution_timeout=0: a finite %.0fs bound has replaced unbounded waiting ' '(deployment=%s).', self._task_queue_config.effective_execution_timeout, deployment_name or 'unknown') + # The Inline_Fast_Path window must stay strictly under Long_Poll_Window: a + # submit that peeks longer than a retrieve would wait makes no sense (D4). + # Raised, not asserted -- `python -O` strips asserts and would drop this + # invariant silently. + _inline = self._task_queue_config.inline_fast_path_timeout + _window = long_poll_window() + if _inline >= _window: + raise ConfigError( + 'inline_fast_path_timeout', + _inline, + message=(f'inline_fast_path_timeout ({_inline}s) must be < Long_Poll_Window ' + f'({_window}s); lower it or raise TWINKLE_LONG_POLL_TIMEOUT.')) self._deployment_name = deployment_name self._task_metrics = get_task_metrics(deployment_name) if deployment_name else None self._future_absolute_ttl = self._task_queue_config.absolute_future_ttl(collect_width) @@ -202,74 +217,43 @@ def _queue_key(model_id: str | None, token: str | None) -> str: async def _perform_preflight_checks( self, - request_id: str, model_id: str | None, token: str | None, input_tokens: int, batch_size: int | None = None, data_world_size: int | None = None, batch_size_multiple: int | None = None, - persist_failure: bool = True, - ) -> dict[str, Any] | None: + ) -> None: """Run rate-limit and validation checks before queuing a task. - Returns None if all checks pass, or an error-response dict on failure. + Returns ``None`` when every check passes. On failure it RAISES a + ``RequestRejectedError`` subclass -- the Decision_Boundary is this line, and + raising before any ``store_future_status`` call is what guarantees zero + future writes for a rejected request (Property 3). It writes no FAILED + record and returns no ``_error`` marker. """ if not token or not self._task_queue_config.enabled: - return None - - async def reject(error_msg: str, queue_state: str) -> dict[str, Any]: - error_code = 429 if queue_state == QueueState.PAUSED_RATE_LIMIT.value else 400 - error_payload = task_error_payload( - error_msg, - request_id=request_id, - error_code=error_code, - category=ErrorCategory.User, - ) - if persist_failure: - await self.state.store_future_status( - request_id, - TaskStatus.FAILED.value, - model_id, - result=error_payload, - queue_state=queue_state, - queue_state_reason=error_msg, - replica_id=getattr(self, 'replica_id', None), - ) - return {'request_id': request_id, 'model_id': model_id} - # Private marker consumed by schedule_task_and_wait(). It is not - # returned by the public polling-style schedule_task() API. - return { - 'request_id': request_id, - 'model_id': model_id, - '_error': error_msg, - } + return if input_tokens > self._task_queue_config.max_input_tokens: - error_msg = (f'Input tokens ({input_tokens}) exceed maximum allowed ' - f'({self._task_queue_config.max_input_tokens})') - return await reject(error_msg, QueueState.UNKNOWN.value) + raise InputTokensExceededError(f'Input tokens ({input_tokens}) exceed maximum allowed ' + f'({self._task_queue_config.max_input_tokens})') if batch_size is not None and data_world_size is not None: if batch_size < data_world_size: - error_msg = (f'Batch size {batch_size} must be >= data world size {data_world_size}') - return await reject(error_msg, QueueState.UNKNOWN.value) + raise BatchSizeError(f'Batch size {batch_size} must be >= data world size {data_world_size}') if batch_size_multiple is not None: required_multiple = data_world_size * batch_size_multiple if batch_size % required_multiple != 0: - error_msg = (f'Batch size {batch_size} must be divisible by {required_multiple} ' - f'so each data-parallel shard gets a multiple of ' - f'{batch_size_multiple} examples') - return await reject(error_msg, QueueState.UNKNOWN.value) + raise BatchSizeError(f'Batch size {batch_size} must be divisible by {required_multiple} ' + f'so each data-parallel shard gets a multiple of ' + f'{batch_size_multiple} examples') allowed, reason = await self._rate_limiter.check_and_record(token, input_tokens) if not allowed: if self._task_metrics: self._task_metrics.rate_limit_rejections.inc(tags={'deployment': self._deployment_name}) - error_msg = f'Rate limit exceeded: {reason}' - return await reject(error_msg, QueueState.PAUSED_RATE_LIMIT.value) - - return None + raise RateLimitExceededError(f'Rate limit exceeded: {reason}') async def _schedule_task( self, @@ -281,38 +265,33 @@ async def _schedule_task( data_world_size: int | None = None, batch_size_multiple: int | None = None, task_type: str | None = None, - *, - completion: asyncio.Future[Any] | None = None, - persist_status: bool, + request_id: str | None = None, ) -> dict[str, Any]: - """Common enqueue path for polling and in-process wait callers.""" - request_id = f'req_{uuid.uuid4().hex}' + """Common enqueue path. Always persists status: the future record is the + single delivery channel for both result and failure.""" + request_id = request_id or f'req_{uuid.uuid4().hex}' - preflight_result = await self._perform_preflight_checks( - request_id=request_id, + # Decision_Boundary: raises RequestRejectedError before any state write. + await self._perform_preflight_checks( model_id=model_id, token=token, input_tokens=input_tokens, batch_size=batch_size, data_world_size=data_world_size, batch_size_multiple=batch_size_multiple, - persist_failure=persist_status, ) - if preflight_result is not None: - return preflight_result if self._event_loop is None: self._event_loop = asyncio.get_running_loop() - if persist_status: - await self.state.store_future_status( - request_id, - TaskStatus.PENDING.value, - model_id, - queue_state=QueueState.ACTIVE.value, - replica_id=getattr(self, 'replica_id', None), - absolute_deadline=self._future_deadline(), - ) + await self.state.store_future_status( + request_id, + TaskStatus.PENDING.value, + model_id, + queue_state=QueueState.ACTIVE.value, + replica_id=getattr(self, 'replica_id', None), + absolute_deadline=self._future_deadline(), + ) queue_key = self._queue_key(model_id=model_id, token=token) self._compute_worker.ensure_queue_registered(queue_key) @@ -328,16 +307,13 @@ async def _schedule_task( input_tokens=input_tokens, task_type=task_type, created_at=time.monotonic(), - completion=completion, - persist_status=persist_status, )) - if persist_status: - await self.state.store_future_status( - request_id, - TaskStatus.QUEUED.value, - model_id, - queue_state=QueueState.ACTIVE.value, - ) + await self.state.store_future_status( + request_id, + TaskStatus.QUEUED.value, + model_id, + queue_state=QueueState.ACTIVE.value, + ) logger.info(f'[TaskQueue] Task {request_id} queued, type={task_type or "unknown"}, ' f'model_id={model_id}, queue_key={queue_key}, ' f'queue_depth={q.qsize()}, input_tokens={input_tokens}') @@ -360,6 +336,7 @@ async def schedule_task( data_world_size: int | None = None, batch_size_multiple: int | None = None, task_type: str | None = None, + request_id: str | None = None, ) -> dict[str, Any]: """Schedule a GPU compute task through the serial compute queue. @@ -388,46 +365,66 @@ async def schedule_task( data_world_size=data_world_size, batch_size_multiple=batch_size_multiple, task_type=task_type, - persist_status=True, + request_id=request_id, ) - async def schedule_task_and_wait( + # Poll cadence *inside* the Inline_Fast_Path window. Much smaller than the window + # itself so a task that finishes early is noticed promptly; the window bound + # (config.inline_fast_path_timeout) is what actually caps submit latency. + _INLINE_FAST_PATH_POLL = 0.005 + + async def _peek_terminal(self, request_id: str, *, fallback_status: str) -> TaskEnvelope: + """Poll a record up to the Inline_Fast_Path window; return a terminal envelope + if it settled, else a non-terminal envelope with ``fallback_status``.""" + deadline = time.monotonic() + self._task_queue_config.inline_fast_path_timeout + record = None + while time.monotonic() < deadline: + record = await self.state.get_future(request_id) + if record is not None and record.get('status') in TERMINAL_STATUSES: + return envelope_from_record(request_id, record) + await asyncio.sleep(self._INLINE_FAST_PATH_POLL) + return envelope_from_record(request_id, record, fallback_status=fallback_status) + + async def submit_and_peek( self, coro_factory: Callable[[], Coroutine], + *, model_id: str | None = None, token: str | None = None, - input_tokens: int = 0, - batch_size: int | None = None, - data_world_size: int | None = None, - batch_size_multiple: int | None = None, task_type: str | None = None, - ) -> Any: - """Schedule a compute task and block until it completes. + request_id: str | None = None, + **schedule_kwargs: Any, + ) -> TaskEnvelope: + """Enqueue a task, then briefly wait so a fast op finishes in one round trip. + + Exceeding the window is not an error: the caller polls + ``/twinkle/retrieve_future`` instead. That is what makes this loop + fundamentally different from the deleted in-process blocking wait -- it owes + nothing to failure handling, so it needs no terminal write, no missing-record + branch, and no race with the worker. A terminal record inside the window is + returned as a terminal envelope (success or failure); a window that elapses + still non-terminal returns a non-terminal envelope carrying ``queue_state``. + """ + ref = await self.schedule_task( + coro_factory, model_id=model_id, token=token, task_type=task_type, request_id=request_id, **schedule_kwargs) + return await self._peek_terminal(ref['request_id'], fallback_status='pending') - Twinkle-side counterpart to schedule_task(). Enqueues the task through - the same serial worker but delivers the result through an in-process - Future. Large model outputs therefore never enter ServerState. + async def submit_background_and_peek( + self, + coro_factory: Callable[[], Coroutine], + *, + model_id: str | None = None, + task_type: str | None = None, + ) -> TaskEnvelope: + """Fire-and-forget variant of :meth:`submit_and_peek` for pure-I/O tasks. - Raises: - RuntimeError: If the task fails or scheduling is rejected. + Uses ``schedule_background_task`` (outside the serial compute queue) but returns + the same Task_Envelope, so ``upload_to_hub`` shares the retrieve/future machinery + instead of its own status endpoint. The task is already RUNNING on return, so the + non-terminal fallback is ``running``. """ - completion = asyncio.get_running_loop().create_future() - task_ref = await self._schedule_task( - coro_factory, - model_id=model_id, - token=token, - input_tokens=input_tokens, - batch_size=batch_size, - data_world_size=data_world_size, - batch_size_multiple=batch_size_multiple, - task_type=task_type, - completion=completion, - persist_status=False, - ) - if error := task_ref.get('_error'): - completion.cancel() - raise RuntimeError(error) - return await completion + ref = await self.schedule_background_task(coro_factory, model_id=model_id, task_type=task_type) + return await self._peek_terminal(ref['request_id'], fallback_status='running') async def schedule_background_task( self, diff --git a/src/twinkle/server/utils/task_queue/types.py b/src/twinkle/server/utils/task_queue/types.py index c7f51462..b173b0c1 100644 --- a/src/twinkle/server/utils/task_queue/types.py +++ b/src/twinkle/server/utils/task_queue/types.py @@ -9,11 +9,9 @@ """ from __future__ import annotations -import asyncio from collections.abc import Callable, Coroutine from dataclasses import dataclass from enum import Enum -from typing import Any class TaskStatus(Enum): @@ -23,7 +21,7 @@ class TaskStatus(Enum): RUNNING = 'running' # Task currently executing COMPLETED = 'completed' # Task completed successfully FAILED = 'failed' # Task failed with error - RATE_LIMITED = 'rate_limited' # Task rejected due to rate limiting + CANCELLED = 'cancelled' # Task cancelled by the client before it started running class UserTaskError(ValueError): @@ -63,9 +61,3 @@ class QueuedTask: task_type: str | None created_at: float first_rate_limited_at: float | None = None - # ``schedule_task_and_wait`` is an in-process request/response path. Its - # potentially large result is delivered through this Future instead of - # being persisted in ServerState merely for the same process to read it - # back. Polling-style ``schedule_task`` leaves this as ``None``. - completion: asyncio.Future[Any] | None = None - persist_status: bool = True diff --git a/src/twinkle/server/utils/task_queue/worker.py b/src/twinkle/server/utils/task_queue/worker.py index b1e276cf..b0e5fc14 100644 --- a/src/twinkle/server/utils/task_queue/worker.py +++ b/src/twinkle/server/utils/task_queue/worker.py @@ -28,7 +28,7 @@ logger = get_logger() -# Ray_Get_Timeout is classified the same as asyncio.TimeoutError: 504/Server (R5#8). +# Ray_Get_Timeout is classified the same as asyncio.TimeoutError: 504/Server. try: from ray.exceptions import GetTimeoutError as _RayGetTimeout _TIMEOUT_EXCEPTIONS: tuple[type[BaseException], ...] = (asyncio.TimeoutError, _RayGetTimeout) @@ -60,7 +60,7 @@ def __init__( self._config = config self._task_metrics = task_metrics self._deployment_name = deployment_name - # Optional coroutine-returning callback fired on a backend timeout (R3#2). + # Optional coroutine-returning callback fired on a backend timeout. self._on_backend_timeout = on_backend_timeout self.task_queues: dict[str, asyncio.Queue] = {} @@ -136,16 +136,6 @@ def _record_queue_metrics(self, task_type: str, queue_wait: float) -> None: # ------------------------------------------------------------------ - @staticmethod - def _complete_result(task: QueuedTask, result: Any) -> None: - if task.completion is not None and not task.completion.done(): - task.completion.set_result(result) - - @staticmethod - def _complete_error(task: QueuedTask, error: str) -> None: - if task.completion is not None and not task.completion.done(): - task.completion.set_exception(RuntimeError(error)) - async def _store_task_failed( self, task: QueuedTask, @@ -157,23 +147,26 @@ async def _store_task_failed( category: ErrorCategory = ErrorCategory.Server, traceback_text: str | None = None, ) -> None: - """Store FAILED status with a standardised ``ErrorPayload``.""" - if task.persist_status: - await self._state.store_future_status( - task.request_id, - TaskStatus.FAILED.value, - task.model_id, - result=task_error_payload( - error, - request_id=task.request_id, - error_code=error_code, - category=category, - traceback_text=traceback_text, - ), - queue_state=queue_state, - queue_state_reason=queue_state_reason, - ) - self._complete_error(task, error) + """Store FAILED status with a standardised ``ErrorPayload``. + + The future record is the single delivery channel: a failed task is written + unconditionally so both the Inline_Fast_Path peek and the Retrieve_Endpoint + observe the same terminal record. + """ + await self._state.store_future_status( + task.request_id, + TaskStatus.FAILED.value, + task.model_id, + result=task_error_payload( + error, + request_id=task.request_id, + error_code=error_code, + category=category, + traceback_text=traceback_text, + ), + queue_state=queue_state, + queue_state_reason=queue_state_reason, + ) async def fail_queue_tasks(self, queue_key: str, reason: str) -> None: """Drain a queue and mark all pending tasks as FAILED.""" @@ -224,13 +217,12 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue Handles execution timeout, general exceptions, and always calls q.task_done() in the finally block. """ - if task.persist_status: - await self._state.store_future_status( - task.request_id, - TaskStatus.RUNNING.value, - task.model_id, - queue_state=QueueState.ACTIVE.value, - ) + await self._state.store_future_status( + task.request_id, + TaskStatus.RUNNING.value, + task.model_id, + queue_state=QueueState.ACTIVE.value, + ) task_type = task.task_type or 'unknown' exec_start = time.monotonic() @@ -258,15 +250,13 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue result = await asyncio.wait_for(coro, timeout=self._config.effective_execution_timeout) exec_time = time.monotonic() - exec_start logger.info(f'[ComputeWorker] Task {task.request_id} completed in {exec_time:.2f}s, type={task_type}') - if task.persist_status: - await self._state.store_future_status( - task.request_id, - TaskStatus.COMPLETED.value, - task.model_id, - result=result, - queue_state=QueueState.ACTIVE.value, - ) - self._complete_result(task, result) + await self._state.store_future_status( + task.request_id, + TaskStatus.COMPLETED.value, + task.model_id, + result=result, + queue_state=QueueState.ACTIVE.value, + ) except _TIMEOUT_EXCEPTIONS: task_status = 'timeout' exec_time = time.monotonic() - exec_start @@ -274,10 +264,10 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue f'actual execution time: {exec_time:.2f}s') logger.error(f'[ComputeWorker] Task {task.request_id} TIMEOUT after {exec_time:.2f}s, ' f'type={task_type}, queue_key={queue_key}') - # asyncio.TimeoutError and Ray_Get_Timeout are 504/Server (R5#8). + # asyncio.TimeoutError and Ray_Get_Timeout are 504/Server. await self._store_task_failed(task, error, QueueState.ACTIVE.value, error_code=504) # Probe actor liveness after a timeout so an operator learns the replica's - # state without waiting for a second request to also time out (R3#2). + # state without waiting for a second request to also time out. if self._on_backend_timeout is not None: try: await self._on_backend_timeout() @@ -299,13 +289,13 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue error = str(exc) logger.error(f'[ComputeWorker] Task {task.request_id} REFUSED (admission gate held) after ' f'{exec_time:.2f}s, type={task_type}, queue_key={queue_key}') - # Gate held by a leaked timed-out call -> 503/Server (R2#4). + # Gate held by a leaked timed-out call -> 503/Server. await self._store_task_failed(task, error, QueueState.ACTIVE.value, error_code=503) except Exception as exc: task_status = 'failed' exec_time = time.monotonic() - exec_start # error is a single-line summary; the full traceback goes only to the - # traceback field, never into `error` (R5#7). + # traceback field, never into `error`. error = f'{type(exc).__name__}: {exc}' logger.error(f'[ComputeWorker] Task {task.request_id} FAILED after {exec_time:.2f}s, ' f'type={task_type}:\n{traceback.format_exc(limit=3)}') @@ -346,8 +336,8 @@ async def _try_run_one(self) -> bool: continue # try the next queue # A record already in a Terminal_State (e.g. written 'failed' by the - # state-hygiene orphan handling) must not be executed again (R3#8). - if task.persist_status and await self._is_record_terminal(task.request_id): + # state-hygiene orphan handling) must not be executed again. + if await self._is_record_terminal(task.request_id): logger.info(f'[ComputeWorker] Task {task.request_id} already terminal on dequeue; skipping.') q.task_done() continue @@ -366,7 +356,7 @@ async def _is_record_terminal(self, request_id: str) -> bool: return False if not record: return False - return record.get('status') in (TaskStatus.COMPLETED.value, TaskStatus.FAILED.value) + return record.get('status') in (TaskStatus.COMPLETED.value, TaskStatus.FAILED.value, TaskStatus.CANCELLED.value) # ------------------------------------------------------------------ # Main worker loop diff --git a/src/twinkle_client/_future.py b/src/twinkle_client/_future.py new file mode 100644 index 00000000..65a2343e --- /dev/null +++ b/src/twinkle_client/_future.py @@ -0,0 +1,168 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Client_Future_Layer: the one polling implementation in Twinkle_Client. + +Public client methods keep their synchronous signatures by calling :func:`resolve`; +no future object is ever exposed. Private module (underscore name) because it is +never imported by Twinkle_Server. +""" +from __future__ import annotations + +import logging +import time +from typing import Any, Optional + +import requests + +from twinkle_client.exceptions import TaskCancelledError, TaskFailedError, TaskRecordLostError, TaskWaitTimeoutError +from twinkle_client.http import http_post +from twinkle_client.http.utils import get_base_url +from twinkle_client.types.lifecycle import TaskEnvelope, TERMINAL_STATUSES + +logger = logging.getLogger('twinkle_client') + +# An independent constant, NOT derived from any server-side timeout: the server +# guarantees a task reaches a terminal state, so this is only "how long the client is +# willing to wait". Deliberately different from the server's execution_timeout fallback +# so a reader does not think the two are related. +_DEFAULT_TOTAL_TIMEOUT = 7200.0 + +# A 404 is Retrieve_Endpoint's verdict after a whole Long_Poll_Window, but state +# jitter (Redis blip, actor restart) can hide an in-flight record for one window, +# so bound-retry before declaring the record lost. +_NOT_FOUND_RETRY_MAX = 3 + +# 5xx / 408 / connection errors are retried with exponential backoff. +_TRANSPORT_RETRY_MAX = 5 + + +def _retrieve_url() -> str: + return f'{get_base_url()}/twinkle/retrieve_future' + + +def _cancel_url() -> str: + return f'{get_base_url()}/twinkle/cancel' + + +def _best_effort_cancel(request_id: str) -> None: + """Ask the server to drop a task when the caller abandons the wait (e.g. Ctrl-C). + + Never raises: a failed cancel must not mask the original interrupt. The server + only drops not-yet-started tasks, so a running task is unaffected. + """ + try: + http_post(url=_cancel_url(), json_data={'request_id': request_id}, timeout=2) + except BaseException as e: # noqa: BLE001 - best effort; never mask the interrupt + logger.debug('[future] best-effort cancel of %s failed: %s', request_id, e) + + +def _post_retrieve(request_id: str) -> TaskEnvelope: + """POST one retrieve and parse the reply into a TaskEnvelope. + + Raises ``requests.HTTPError`` (a :class:`TwinkleHTTPError` after the client + error-parsing change lands) on a non-2xx response. + """ + response = http_post(url=_retrieve_url(), json_data={'request_id': request_id}) + return TaskEnvelope.model_validate(response.json()) + + +def _status_of(error: requests.HTTPError) -> Optional[int]: + status = getattr(error, 'status_code', None) + if status is None and getattr(error, 'response', None) is not None: + status = error.response.status_code + return status + + +def _is_retryable(status: int) -> bool: + return status == 408 or 500 <= status <= 599 + + +def _log_queue_state(reply: TaskEnvelope) -> None: + if reply.queue_state and reply.queue_state != 'active': + logger.info('[future] task %s waiting: queue_state=%s reason=%s', reply.request_id, reply.queue_state, + reply.queue_state_reason) + + +def _unwrap(env: TaskEnvelope, model_cls) -> Any: + """Turn a terminal TaskEnvelope into a return value or an exception. + + Takes the envelope *whole* rather than destructured fields: the submit and + retrieve paths must not be able to pass different subsets. A signature like + ``(status, result, request_id, model_cls, error=None)`` would let the submit + path simply never pass ``error`` -- and then every failure completing inside + the Inline_Fast_Path window would raise 'no recorded payload' while its real + payload sat unread. + """ + if env.status == 'failed': + p = env.error + raise TaskFailedError( + p.error, + category=p.category.value, + request_id=env.request_id, + error_code=p.error_code, + details=p.details, + ) + if env.status == 'cancelled': + p = env.error + raise TaskCancelledError( + p.error if p is not None else 'Task cancelled', + request_id=env.request_id, + error_code=p.error_code if p is not None else None, + ) + return model_cls.model_validate(env.result) if model_cls is not None else env.result + + +def resolve(submit: TaskEnvelope, *, model_cls, total_timeout: float = _DEFAULT_TOTAL_TIMEOUT) -> Any: + """Block until ``submit``'s task reaches a terminal state, then return its result. + + A terminal submit envelope is unwrapped directly, issuing no Retrieve_Endpoint + request at all (single round trip for control-plane ops). Otherwise the same + ``_unwrap`` is applied to each retrieve reply, so a failure inside the + Inline_Fast_Path window and one observed via retrieve take an identical path. + + The main loop never sleeps: waiting is delegated to Retrieve_Endpoint's + long-poll. Only transport retries back off. + """ + if submit.status in TERMINAL_STATUSES: + return _unwrap(submit, model_cls) # same call as the retrieve path + + deadline = time.monotonic() + total_timeout + transport_failures = not_found_count = 0 + try: + while True: + if time.monotonic() >= deadline: + raise TaskWaitTimeoutError(request_id=submit.request_id, waited=total_timeout) + try: + reply = _post_retrieve(submit.request_id) + transport_failures = not_found_count = 0 + except requests.HTTPError as e: + status = _status_of(e) + if status == 404: + not_found_count += 1 + if not_found_count > _NOT_FOUND_RETRY_MAX: + raise TaskRecordLostError(request_id=submit.request_id) from e + continue + if status is None or not _is_retryable(status): + raise + transport_failures += 1 + if transport_failures > _TRANSPORT_RETRY_MAX: + raise + time.sleep(min(2 ** transport_failures, 30)) + continue + if reply.status in TERMINAL_STATUSES: + return _unwrap(reply, model_cls) # same call as the submit path + _log_queue_state(reply) + except (KeyboardInterrupt, SystemExit): + # Caller abandoned the wait: best-effort ask the server to drop the task if it + # has not started, then re-raise so the interrupt is never swallowed. + _best_effort_cancel(submit.request_id) + raise + + +def resolve_response(response, model_cls) -> Any: + """Resolve a Submit_Endpoint HTTP response through the Client_Future_Layer. + + The one place the (already status-checked) reply's Task_Envelope is validated + and resolved, shared by every public client method so they keep synchronous + signatures without duplicating the parse+resolve step. + """ + return resolve(TaskEnvelope.model_validate(response.json()), model_cls=model_cls) diff --git a/src/twinkle_client/exceptions.py b/src/twinkle_client/exceptions.py new file mode 100644 index 00000000..86bdc3d2 --- /dev/null +++ b/src/twinkle_client/exceptions.py @@ -0,0 +1,116 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Twinkle_Client exceptions for the request lifecycle. + +Two axes, kept deliberately distinct: + +- Transport / HTTP failures (:class:`TwinkleHTTPError`) inherit ``requests.HTTPError`` + so existing ``except requests.HTTPError`` clauses keep working. They carry the + server's ``error_code`` / ``category`` when the response body had them. +- Task-outcome and polling failures (:class:`TaskFailedError`, + :class:`TaskWaitTimeoutError`, :class:`TaskRecordLostError`) do NOT inherit + ``requests.HTTPError``: a task that reaches a ``failed`` terminal state is + delivered over HTTP 200, so it is not an HTTP-level error. +""" +from __future__ import annotations + +from typing import Any, Optional + +import requests + + +class TwinkleHTTPError(requests.HTTPError): + """An HTTP 4xx/5xx (other than 410) from a twinkle endpoint. + + Inherits ``requests.HTTPError`` so callers already catching that keep working. + ``status_code`` is the HTTP status; ``error_code`` / ``category`` come from the + server's structured error body when present (else ``None`` / ``'Unknown'``). + """ + + def __init__( + self, + *args: Any, + status_code: Optional[int] = None, + error_code: Optional[int] = None, + category: str = 'Unknown', + request_id: Optional[str] = None, + **kwargs: Any, + ) -> None: + super().__init__(*args, **kwargs) + self.status_code = status_code + self.error_code = error_code + self.category = category + self.request_id = request_id + + +class TaskFailedError(Exception): + """A task reached the ``failed`` terminal state (delivered over HTTP 200). + + The twinkle counterpart of tinker's ``RequestFailedError`` + (``tinker/_exceptions.py``; carries ``message`` / ``request_id`` / ``category``). + Deliberately NOT a ``requests.HTTPError`` subclass: the HTTP call succeeded, it + is the *task* that failed, so this is not an HTTP-level error. + """ + + def __init__( + self, + error: str, + *, + category: str, + request_id: str, + error_code: Optional[int] = None, + details: Optional[list[dict[str, Any]]] = None, + ) -> None: + super().__init__(error) + self.error = error + self.category = category + self.request_id = request_id + self.error_code = error_code + self.details = details + + +class TaskCancelledError(Exception): + """A task reached the ``cancelled`` terminal state (delivered over HTTP 200). + + Distinct from :class:`TaskFailedError`: the task did not fail, it was cancelled + before it started running (client cancel). Not a ``requests.HTTPError`` -- the + HTTP call succeeded; the task was simply dropped. + """ + + def __init__( + self, + error: str, + *, + request_id: str, + error_code: Optional[int] = None, + ) -> None: + super().__init__(error) + self.error = error + self.request_id = request_id + self.error_code = error_code + + +class TaskWaitTimeoutError(Exception): + """The Client_Future_Layer stopped polling after ``total_timeout`` seconds. + + "I am not waiting any longer" -- distinct from :class:`TaskRecordLostError`, + which indicates a state-layer problem. The task itself is guaranteed to reach a + terminal state by the server; this only means the client gave up. + """ + + def __init__(self, *, request_id: str, waited: float) -> None: + super().__init__(f'Timed out after {waited:.1f}s waiting for task {request_id}') + self.request_id = request_id + self.waited = waited + + +class TaskRecordLostError(Exception): + """Retrieve_Endpoint returned 404 for a whole run of consecutive attempts. + + Distinct from :class:`TaskWaitTimeoutError`: a 404 run points at the state layer + (Redis blip, actor restart) rather than a slow task, so the operator response + differs. + """ + + def __init__(self, *, request_id: str) -> None: + super().__init__(f'Task record for {request_id} was not found after repeated retries') + self.request_id = request_id diff --git a/src/twinkle_client/http/http_utils.py b/src/twinkle_client/http/http_utils.py index 6aac84a1..3fc71c05 100644 --- a/src/twinkle_client/http/http_utils.py +++ b/src/twinkle_client/http/http_utils.py @@ -1,9 +1,16 @@ import requests -from typing import Any, Callable, Dict, Mapping, Optional +from typing import Any, Dict, Optional +from twinkle_client.exceptions import TwinkleHTTPError from .headers import build_routing_headers from .utils import get_api_key, get_base_url, get_request_id, get_session_id +# Single shared HTTP timeout for every client request (was three separate 600s). +# Must be <= 120 and strictly greater than the server Long_Poll_Window (default 30), +# so a retrieve that waits a full window still completes within the timeout and, being +# < a typical 60s gateway idle limit, survives the gateway. +_HTTP_TIMEOUT = 90 + def _build_headers(additional_headers: Optional[Dict[str, str]] = None) -> Dict[str, str]: """ @@ -49,36 +56,45 @@ def _serialize_params(params: Dict[str, Any]) -> Dict[str, Any]: def _handle_response(response: requests.Response) -> requests.Response: - """ - Handle common response processing. - - Args: - response: Response object - - Returns: - Response object + """Handle common response processing. Raises: - StopIteration: When server returns HTTP 410 (iterator exhausted) - requests.HTTPError: When server returns a 4xx/5xx error, with the - server-side ``detail`` field (full traceback) included in the - exception message so callers don't need to inspect the response body. + StopIteration: When server returns HTTP 410 (iterator exhausted). + TwinkleHTTPError: When the server returns a 4xx/5xx (other than 410). It + inherits ``requests.HTTPError`` so existing ``except`` clauses keep + working, and carries the server's top-level ``error_code`` / ``category`` + / ``request_id`` when present. When those fields are absent (FastAPI's + built-in 404/405, or a gateway passthrough), it falls back to ``detail`` + with ``category='Unknown'``. """ - # Convert HTTP 410 Gone to StopIteration - # This indicates an iterator has been exhausted + # Convert HTTP 410 Gone to StopIteration (an iterator has been exhausted). if response.status_code == 410: raise StopIteration(response.json().get('detail', 'Iterator exhausted')) if not response.ok: try: - detail = response.json().get('detail', response.text) + body = response.json() except Exception: - detail = response.text + body = None + if isinstance(body, dict): + category = body.get('category', 'Unknown') + error_code = body.get('error_code') + request_id = body.get('request_id') + summary = body.get('error') or body.get('detail') or response.text + else: + category, error_code, request_id, summary = 'Unknown', None, None, response.text http_error_msg = ( f'{response.status_code} Error for url: {response.url}\n' - f'Server detail:\n{detail}' + f'Server detail:\n{summary}' + ) + raise TwinkleHTTPError( + http_error_msg, + response=response, + status_code=response.status_code, + error_code=error_code, + category=category, + request_id=request_id, ) - raise requests.HTTPError(http_error_msg, response=response) return response @@ -87,7 +103,7 @@ def http_get( url: Optional[str] = None, params: Optional[Dict[str, Any]] = {}, additional_headers: Optional[Dict[str, str]] = {}, - timeout: int = 600, + timeout: int = _HTTP_TIMEOUT, ) -> requests.Response: """ Send HTTP GET request with required headers. @@ -120,7 +136,7 @@ def http_post( json_data: Optional[Dict[str, Any]] = {}, data: Optional[Any] = {}, additional_headers: Optional[Dict[str, str]] = {}, - timeout: Optional[int] = 600, + timeout: Optional[int] = _HTTP_TIMEOUT, ) -> requests.Response: """ Send HTTP POST request with required headers. @@ -157,7 +173,7 @@ def http_delete( url: Optional[str] = None, params: Optional[Dict[str, Any]] = {}, additional_headers: Optional[Dict[str, str]] = {}, - timeout: int = 600, + timeout: int = _HTTP_TIMEOUT, ) -> requests.Response: """ Send HTTP DELETE request with required headers. diff --git a/src/twinkle_client/model/multi_lora_transformers.py b/src/twinkle_client/model/multi_lora_transformers.py index 3471ca6c..0416acac 100644 --- a/src/twinkle_client/model/multi_lora_transformers.py +++ b/src/twinkle_client/model/multi_lora_transformers.py @@ -1,7 +1,9 @@ from typing import Any, Dict, Optional +import itertools +import logging +import threading from pathlib import Path -import time -from twinkle_client.http import http_get, http_post +from twinkle_client.http import http_post from twinkle_client.common.json_utils import json_safe from twinkle_client.types.component import DataRef from twinkle_client.types.model import ( @@ -10,12 +12,13 @@ ClipGradNormResponse, ForwardBackwardResponse, ForwardResponse, - GetStateDictResponse, GetTrainConfigsResponse, SaveResponse, TrainingProgressResponse, ) +logger = logging.getLogger('twinkle_client') + def _data_ref_payload(inputs: DataRef | list[DataRef]) -> dict[str, Any]: """Encode one or more opaque references for a DataPlane model endpoint.""" @@ -45,11 +48,32 @@ def __init__(self, model_id: str, **kwargs): self.model_id = model_id self.server_url = f'{self.server_url}/model/{model_id}/twinkle' self.adapter_name = None + # Per-client monotonic sequence for idempotent dedup of stateful training ops: + # the server dedups on (session_id, seq_id) so a retried grad/step call is + # applied at most once. Reserved once per call and reused on retry. + self._seq_counter = itertools.count(1) + self._seq_lock = threading.Lock() response = http_post( url=f'{self.server_url}/create', ) response.raise_for_status() + @staticmethod + def _await_task(response, model_cls): + """Resolve a Submit_Endpoint response through the Client_Future_Layer. + + Blocks until the task is terminal and returns the deserialized ``model_cls`` + result (or ``None``), raising ``TaskFailedError`` on a failed terminal state. + Keeps every public method's synchronous signature unchanged. + """ + from twinkle_client._future import resolve_response + return resolve_response(response, model_cls) + + def _next_seq_id(self) -> int: + """Reserve the next monotonic seq_id for a stateful op (dedup key with session).""" + with self._seq_lock: + return next(self._seq_counter) + def add_adapter_to_model(self, adapter_name: str, config: Optional[Dict[str, Any]] = None, **kwargs) -> None: """Add a new adapter to the model. @@ -64,7 +88,7 @@ def add_adapter_to_model(self, adapter_name: str, config: Optional[Dict[str, Any url=f'{self.server_url}/add_adapter_to_model', json_data={'adapter_name': adapter_name, 'config': config, **kwargs} ) - response.raise_for_status() + self._await_task(response, None) self.adapter_name = adapter_name def remove_adapter(self, adapter_name: str | None = None) -> None: @@ -74,7 +98,7 @@ def remove_adapter(self, adapter_name: str | None = None) -> None: url=f'{self.server_url}/remove_adapter', json_data={'adapter_name': name}, ) - response.raise_for_status() + self._await_task(response, None) if name == self.adapter_name: self.adapter_name = None @@ -84,8 +108,7 @@ def forward(self, inputs: Any, **kwargs) -> ForwardResponse: url=f'{self.server_url}/forward', json_data={'inputs': inputs, 'adapter_name': self.adapter_name, **kwargs}, ) - response.raise_for_status() - return ForwardResponse(**response.json()) + return self._await_task(response, ForwardResponse) def forward_only(self, inputs: Any, **kwargs) -> ForwardResponse: """Execute forward pass without gradient computation on inline inputs.""" @@ -93,8 +116,7 @@ def forward_only(self, inputs: Any, **kwargs) -> ForwardResponse: url=f'{self.server_url}/forward_only', json_data={'inputs': inputs, 'adapter_name': self.adapter_name, **kwargs}, ) - response.raise_for_status() - return ForwardResponse(**response.json()) + return self._await_task(response, ForwardResponse) def forward_from_data_plane( self, @@ -115,8 +137,7 @@ def forward_from_data_plane( **json_safe(kwargs), }, ) - response.raise_for_status() - return ForwardResponse(**response.json()) + return self._await_task(response, ForwardResponse) def forward_only_from_data_plane( self, @@ -142,8 +163,7 @@ def forward_only_from_data_plane( url=f'{self.server_url}/forward_only_from_data_plane', json_data=body, ) - response.raise_for_status() - result = ForwardResponse(**response.json()) + result = self._await_task(response, ForwardResponse) if output_ref is not None: return DataRef(**result.result) return result @@ -154,8 +174,7 @@ def calculate_loss(self, **kwargs) -> CalculateLossResponse: url=f'{self.server_url}/calculate_loss', json_data={'adapter_name': self.adapter_name, **kwargs} ) - response.raise_for_status() - return CalculateLossResponse(**response.json()) + return self._await_task(response, CalculateLossResponse) def get_train_configs(self, **kwargs) -> GetTrainConfigsResponse: """Get training configs.""" @@ -163,25 +182,23 @@ def get_train_configs(self, **kwargs) -> GetTrainConfigsResponse: url=f'{self.server_url}/get_train_configs', json_data={'adapter_name': self.adapter_name, **kwargs} ) - response.raise_for_status() - return GetTrainConfigsResponse(**response.json()) + return self._await_task(response, GetTrainConfigsResponse) def backward(self, **kwargs) -> None: """Execute backward pass.""" response = http_post( url=f'{self.server_url}/backward', - json_data={'adapter_name': self.adapter_name, **kwargs} + json_data={'adapter_name': self.adapter_name, 'seq_id': self._next_seq_id(), **kwargs} ) - response.raise_for_status() + self._await_task(response, None) def forward_backward(self, inputs: Any, **kwargs) -> ForwardBackwardResponse: """Execute combined forward and backward pass on inline inputs.""" response = http_post( url=f'{self.server_url}/forward_backward', - json_data={'inputs': inputs, 'adapter_name': self.adapter_name, **kwargs}, + json_data={'inputs': inputs, 'adapter_name': self.adapter_name, 'seq_id': self._next_seq_id(), **kwargs}, ) - response.raise_for_status() - return ForwardBackwardResponse(**response.json()) + return self._await_task(response, ForwardBackwardResponse) def forward_backward_from_data_plane( self, @@ -199,19 +216,19 @@ def forward_backward_from_data_plane( 'adapter_name': self.adapter_name, 'input_field': input_field, 'kwarg_fields': kwarg_fields or {}, + 'seq_id': self._next_seq_id(), **json_safe(kwargs), }, ) - response.raise_for_status() - return ForwardBackwardResponse(**response.json()) + return self._await_task(response, ForwardBackwardResponse) def step(self, **kwargs) -> None: """Execute optimizer step.""" response = http_post( url=f'{self.server_url}/step', - json_data={'adapter_name': self.adapter_name, **kwargs} + json_data={'adapter_name': self.adapter_name, 'seq_id': self._next_seq_id(), **kwargs} ) - response.raise_for_status() + self._await_task(response, None) def zero_grad(self, **kwargs) -> None: """Zero out gradients.""" @@ -219,32 +236,42 @@ def zero_grad(self, **kwargs) -> None: url=f'{self.server_url}/zero_grad', json_data={'adapter_name': self.adapter_name, **kwargs} ) - response.raise_for_status() + self._await_task(response, None) def lr_step(self, **kwargs) -> None: """Execute learning rate scheduler step.""" response = http_post( url=f'{self.server_url}/lr_step', - json_data={'adapter_name': self.adapter_name, **kwargs} + json_data={'adapter_name': self.adapter_name, 'seq_id': self._next_seq_id(), **kwargs} ) - response.raise_for_status() + self._await_task(response, None) def clip_grad_norm(self, max_grad_norm: float = 1.0, norm_type: int = 2, **kwargs) -> ClipGradNormResponse: """Clip gradient norm.""" response = http_post( url=f'{self.server_url}/clip_grad_norm', - json_data={'max_grad_norm': max_grad_norm, 'norm_type': norm_type, 'adapter_name': self.adapter_name, **kwargs} + json_data={ + 'max_grad_norm': max_grad_norm, + 'norm_type': norm_type, + 'adapter_name': self.adapter_name, + **kwargs + } ) - response.raise_for_status() - return ClipGradNormResponse(**response.json()) + return self._await_task(response, ClipGradNormResponse) def clip_grad_and_step(self, max_grad_norm: float = 1.0, norm_type: int = 2, **kwargs) -> None: """Clip gradient norm and execute optimizer step in one call.""" response = http_post( url=f'{self.server_url}/clip_grad_and_step', - json_data={'max_grad_norm': max_grad_norm, 'norm_type': norm_type, 'adapter_name': self.adapter_name, **kwargs} + json_data={ + 'max_grad_norm': max_grad_norm, + 'norm_type': norm_type, + 'adapter_name': self.adapter_name, + 'seq_id': self._next_seq_id(), + **kwargs + } ) - response.raise_for_status() + self._await_task(response, None) def set_loss(self, loss_cls: str, **kwargs) -> None: """Set the loss function.""" @@ -252,7 +279,7 @@ def set_loss(self, loss_cls: str, **kwargs) -> None: url=f'{self.server_url}/set_loss', json_data={'loss_cls': loss_cls, 'adapter_name': self.adapter_name, **kwargs} ) - response.raise_for_status() + self._await_task(response, None) def set_optimizer(self, optimizer_cls: str, **kwargs) -> None: """Set the optimizer.""" @@ -260,7 +287,7 @@ def set_optimizer(self, optimizer_cls: str, **kwargs) -> None: url=f'{self.server_url}/set_optimizer', json_data={'optimizer_cls': optimizer_cls, 'adapter_name': self.adapter_name, **kwargs} ) - response.raise_for_status() + self._await_task(response, None) def set_lr_scheduler(self, scheduler_cls: str, **kwargs) -> None: """Set the learning rate scheduler.""" @@ -268,7 +295,7 @@ def set_lr_scheduler(self, scheduler_cls: str, **kwargs) -> None: url=f'{self.server_url}/set_lr_scheduler', json_data={'scheduler_cls': scheduler_cls, 'adapter_name': self.adapter_name, **kwargs} ) - response.raise_for_status() + self._await_task(response, None) def save(self, name: str, **kwargs) -> SaveResponse: """Save model checkpoint.""" @@ -276,8 +303,7 @@ def save(self, name: str, **kwargs) -> SaveResponse: url=f'{self.server_url}/save', json_data={'name': name, 'adapter_name': self.adapter_name, **kwargs} ) - response.raise_for_status() - return SaveResponse(**response.json()) + return self._await_task(response, SaveResponse) def load(self, name: str, **kwargs) -> None: """Load model checkpoint.""" @@ -285,7 +311,7 @@ def load(self, name: str, **kwargs) -> None: url=f'{self.server_url}/load', json_data={'name': name, 'adapter_name': self.adapter_name, **kwargs} ) - response.raise_for_status() + self._await_task(response, None) def resume_from_checkpoint(self, name: str, *, resume_only_model: bool = False, **kwargs) -> Dict[str, Any]: response = http_post( @@ -293,8 +319,7 @@ def resume_from_checkpoint(self, name: str, *, resume_only_model: bool = False, json_data={'name': name, 'adapter_name': self.adapter_name, 'resume_only_model': resume_only_model, **kwargs} ) - response.raise_for_status() - return TrainingProgressResponse(**response.json()).result + return self._await_task(response, TrainingProgressResponse).result def apply_patch(self, patch_cls: str, **kwargs) -> None: """Apply a patch to the model.""" @@ -302,23 +327,33 @@ def apply_patch(self, patch_cls: str, **kwargs) -> None: url=f'{self.server_url}/apply_patch', json_data={'patch_cls': patch_cls, 'adapter_name': self.adapter_name, **kwargs} ) - response.raise_for_status() + self._await_task(response, None) def add_metric(self, metric_cls: str, is_training: Optional[bool] = None, **kwargs) -> None: """Add a metric to the model.""" response = http_post( url=f'{self.server_url}/add_metric', - json_data={'metric_cls': metric_cls, 'is_training': is_training, 'adapter_name': self.adapter_name, **kwargs} + json_data={ + 'metric_cls': metric_cls, + 'is_training': is_training, + 'adapter_name': self.adapter_name, + **kwargs + } ) - response.raise_for_status() + self._await_task(response, None) def set_template(self, template_cls: str, **kwargs) -> None: """Set the template for data processing.""" response = http_post( url=f'{self.server_url}/set_template', - json_data={'template_cls': template_cls, 'adapter_name': self.adapter_name, 'model_id': self.model_id, **kwargs} + json_data={ + 'template_cls': template_cls, + 'adapter_name': self.adapter_name, + 'model_id': self.model_id, + **kwargs + } ) - response.raise_for_status() + self._await_task(response, None) def set_processor(self, processor_cls: str, **kwargs) -> None: """Set the input processor.""" @@ -326,7 +361,7 @@ def set_processor(self, processor_cls: str, **kwargs) -> None: url=f'{self.server_url}/set_processor', json_data={'processor_cls': processor_cls, 'adapter_name': self.adapter_name, **kwargs} ) - response.raise_for_status() + self._await_task(response, None) def calculate_metric(self, is_training: bool = True, **kwargs) -> CalculateMetricResponse: """Calculate metrics from model outputs.""" @@ -334,17 +369,7 @@ def calculate_metric(self, is_training: bool = True, **kwargs) -> CalculateMetri url=f'{self.server_url}/calculate_metric', json_data={'is_training': is_training, 'adapter_name': self.adapter_name, **kwargs} ) - response.raise_for_status() - return CalculateMetricResponse(**response.json()) - - def get_state_dict(self, **kwargs) -> GetStateDictResponse: - """Get model state dictionary.""" - response = http_post( - url=f'{self.server_url}/get_state_dict', - json_data={'adapter_name': self.adapter_name, **kwargs} - ) - response.raise_for_status() - return GetStateDictResponse(**response.json()) + return self._await_task(response, CalculateMetricResponse) def upload_to_hub( self, @@ -356,16 +381,17 @@ def upload_to_hub( ) -> None: """Upload model checkpoint to hub. - Submits the upload task to the server and polls for completion. - Blocks until the upload finishes or raises on failure. + Submits the upload task and blocks (via the Client_Future_Layer) until it + finishes, raising ``TaskFailedError`` on failure. Args: checkpoint_dir: The directory path of the checkpoint to upload. hub_model_id: The hub model id. hub_token: The hub token (optional). async_upload: Deprecated, has no effect. The server always runs the - upload in the background and the client polls for completion. - poll_interval: Seconds between status poll requests (default: 5). + upload in the background and the client waits via the future layer. + poll_interval: Deprecated, has no effect. Pacing is now owned by the + server-side long-poll of the Retrieve_Endpoint. """ response = http_post( url=f'{self.server_url}/upload_to_hub', @@ -375,23 +401,6 @@ def upload_to_hub( 'hub_token': hub_token, } ) - response.raise_for_status() - request_id = response.json().get('request_id') - if not request_id: - return - - print(f'[upload_to_hub] Upload started (task {request_id}), waiting for completion...') - while True: - status_resp = http_get(url=f'{self.server_url}/upload_status/{request_id}') - status_resp.raise_for_status() - data = status_resp.json() - status = data.get('status', 'unknown') - if status == 'completed': - print(f'[upload_to_hub] Upload completed successfully.') - return - elif status == 'failed': - error = data.get('error', 'Unknown error') - raise RuntimeError(f'[upload_to_hub] Upload failed: {error}') - else: - print(f'[upload_to_hub] Status: {status}...') - time.sleep(poll_interval) + logger.info('[upload_to_hub] upload submitted, waiting for completion...') + self._await_task(response, None) + logger.info('[upload_to_hub] upload completed successfully.') diff --git a/src/twinkle_client/sampler/vllm_sampler.py b/src/twinkle_client/sampler/vllm_sampler.py index 4e271ce7..719b9d06 100644 --- a/src/twinkle_client/sampler/vllm_sampler.py +++ b/src/twinkle_client/sampler/vllm_sampler.py @@ -2,7 +2,8 @@ from dataclasses import asdict from typing import Any, Dict, List, Optional, Union from twinkle_client.http import http_post -from twinkle_client.types.sampler import AddAdapterResponse, SampleResponseModel, SetTemplateResponse +from twinkle_client.types.sampler import (AddAdapterResponse, SampleResponseModel, SampleResponseModelList, + SetTemplateResponse) from peft import PeftConfig from twinkle.data_format import Trajectory, InputFeature, SamplingParams from twinkle_client.common.json_utils import json_safe @@ -49,6 +50,17 @@ def __init__(self, model_id: str, **kwargs): ) response.raise_for_status() + @staticmethod + def _await_task(response, model_cls): + """Resolve a Submit_Endpoint response through the Client_Future_Layer. + + Blocks until the task is terminal and returns the deserialized ``model_cls`` + result (or ``None``), raising ``TaskFailedError`` on a failed terminal state. + Keeps every public method's synchronous signature unchanged. + """ + from twinkle_client._future import resolve_response + return resolve_response(response, model_cls) + def add_adapter_to_sampler(self, adapter_name: str, config: PeftConfig, **kwargs) -> AddAdapterResponse: """Add a new adapter to the sampler.""" if isinstance(config, PeftConfig): @@ -99,8 +111,7 @@ def sample( url=f'{self.server_url}/sample', json_data=json_data ) - response.raise_for_status() - return [SampleResponseModel(**r) for r in response.json()['samples']] + return self._await_task(response, SampleResponseModelList).samples def sample_to_data_plane( self, @@ -128,8 +139,7 @@ def sample_to_data_plane( url=f'{self.server_url}/sample_to_data_plane', json_data=json_safe(body), ) - response.raise_for_status() - return DataRef(**response.json()) + return self._await_task(response, DataRef) async def asample( self, @@ -188,7 +198,7 @@ def set_template(self, template_cls: str, adapter_name: str = '', **kwargs) -> S ) response.raise_for_status() return SetTemplateResponse(**response.json()) - + def apply_patch(self, patch_cls: str, **kwargs) -> None: """Apply a patch to the model.""" response = http_post( diff --git a/src/twinkle_client/types/__init__.py b/src/twinkle_client/types/__init__.py index 1c25324a..35cbfbaf 100644 --- a/src/twinkle_client/types/__init__.py +++ b/src/twinkle_client/types/__init__.py @@ -1,7 +1,6 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from .model import ( AddAdapterRequest, - AddAdapterResponse, AddMetricRequest, AddMetricResponse, AdapterRequest, @@ -23,7 +22,6 @@ ForwardRequest, ForwardResponse, GetStateDictRequest, - GetStateDictResponse, GetTrainConfigsResponse, LoadRequest, LoadResponse, @@ -46,8 +44,6 @@ StepResponse, TrainingProgressResponse, UploadToHubRequest, - UploadToHubResponse, - UploadStatusResponse, ZeroGradResponse, ) from .processor import ( @@ -94,6 +90,7 @@ ) from .checkpoint import ResolvedLoadPath +from .lifecycle import CancelRequest, CancelResponse, RetrieveFutureRequest, TaskEnvelope, TaskStatus, TERMINAL_STATUSES from .component import ( DataAppendRequest, DataGetRequest, diff --git a/src/twinkle_client/types/lifecycle.py b/src/twinkle_client/types/lifecycle.py new file mode 100644 index 00000000..ecff212a --- /dev/null +++ b/src/twinkle_client/types/lifecycle.py @@ -0,0 +1,83 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""The request-lifecycle wire model: one envelope for submit and retrieve. + +This module is a public-contract carrier imported across packages (Twinkle_Server +reverse-imports ``twinkle_client.types``); per the naming rulings in ``base.py`` it +therefore intentionally carries **no** underscore prefix. +""" +from __future__ import annotations + +from typing import Any, Literal, Optional + +from .base import ResponseModel, StrictRequest +from .errors import ErrorPayload, QueueStateLiteral + +# The lifecycle status of a queued task. Kept in sync with the server-side +# ``TaskStatus`` enum values (a consistency test asserts the two sets are equal). +TaskStatus = Literal['pending', 'queued', 'running', 'completed', 'failed', 'cancelled'] + +# The two states past which a task never changes again. ``frozenset`` so a caller +# cannot mutate the shared set. +TERMINAL_STATUSES: frozenset[str] = frozenset({'completed', 'failed', 'cancelled'}) + + +class RetrieveFutureRequest(StrictRequest): + """Body of ``POST /twinkle/retrieve_future``. + + ``request_id`` is the only field: the caller already knows which adapter it + targeted, so no ``model_id`` is needed to correlate the reply. A brand-new + endpoint with no legacy clients, so it takes the strict base (unknown fields + fail loudly) rather than tolerating extras. + """ + + request_id: str + + +class CancelRequest(StrictRequest): + """Body of ``POST /twinkle/cancel``: best-effort cancel of a not-yet-started task. + + New endpoint with no legacy clients, so it takes the strict base (unknown fields + fail loudly). + """ + + request_id: str + + +class CancelResponse(ResponseModel): + """Reply to ``POST /twinkle/cancel``. + + ``cancelled`` is True only when the task is in the terminal ``cancelled`` state + after the attempt; ``state`` is its status afterwards (``cancelled`` / ``running`` + / ``completed`` / ``failed`` / ``not_found``). A running or already-terminal task + is never interrupted -- cancel only drops tasks that have not started. + """ + + cancelled: bool + state: str + + +class TaskEnvelope(ResponseModel): + """The one lifecycle reply, shared by Submit_Endpoint and Retrieve_Endpoint. + + Success and failure live in *different* fields, and both endpoints fill the + same field for the same meaning. That is the whole point: if failure rode in + ``result`` on submit but in ``error`` on retrieve, a task that fails inside the + Inline_Fast_Path window -- which is exactly where ``step`` / ``zero_grad`` / + ``set_loss`` fail -- would have its payload read from the wrong place and + silently dropped. + + ``result`` is ``Optional[Any]`` rather than each endpoint's concrete response + model: this is the lifecycle-layer model, not a per-endpoint generic. + Deserialization to the concrete model is done by the Client_Future_Layer once + it holds a terminal envelope, since it knows the caller's expected type. + + No ``model_id``: the caller already knows which adapter it targeted; + ``request_id`` is the only key needed to correlate a reply. + """ + + request_id: str + status: TaskStatus + result: Optional[Any] = None # set iff status == 'completed' + error: Optional[ErrorPayload] = None # set iff status == 'failed' + queue_state: Optional[QueueStateLiteral] = None + queue_state_reason: Optional[str] = None diff --git a/src/twinkle_client/types/model.py b/src/twinkle_client/types/model.py index 3d4f4cf4..bb8b7db8 100644 --- a/src/twinkle_client/types/model.py +++ b/src/twinkle_client/types/model.py @@ -19,6 +19,8 @@ class Config: class ForwardRequest(BaseModel): inputs: Any adapter_name: str + # Optional monotonic dedup key; only set by grad-mutating client calls (forward_backward). + seq_id: Optional[int] = None class Config: extra = 'allow' @@ -37,6 +39,8 @@ class DataPlaneForwardRequest(BaseModel): input_field: str | None = None kwarg_fields: Dict[str, str] = Field(default_factory=dict) adapter_name: str + # Optional monotonic dedup key; only set by grad-mutating data-plane calls. + seq_id: Optional[int] = None class Config: extra = 'allow' @@ -55,6 +59,8 @@ def validate_output(self) -> 'DataPlaneForwardOnlyRequest': class AdapterRequest(BaseModel): adapter_name: str + # Optional monotonic dedup key; only set by grad-mutating client calls (backward/step/lr_step). + seq_id: Optional[int] = None class Config: extra = 'allow' @@ -176,6 +182,8 @@ class ClipGradAndStepRequest(BaseModel): adapter_name: str max_grad_norm: float = 1.0 norm_type: int = 2 + # Optional monotonic dedup key; set by the grad-mutating clip_grad_and_step call. + seq_id: Optional[int] = None class Config: extra = 'allow' @@ -240,11 +248,6 @@ class GetTrainConfigsResponse(BaseModel): result: str -class GetStateDictResponse(BaseModel): - """Response for /get_state_dict endpoint (returns Dict).""" - result: Dict[str, Any] - - class CalculateMetricResponse(BaseModel): """Response for /calculate_metric endpoint (returns Dict).""" result: Dict[str, Any] @@ -313,18 +316,6 @@ class SetProcessorResponse(OkResponse): pass -class UploadToHubResponse(BaseModel): - """Response for /upload_to_hub endpoint.""" - request_id: str - - -class UploadStatusResponse(BaseModel): - """Response for /upload_status/{request_id} endpoint.""" - request_id: str - status: str # pending / queued / running / completed / failed - error: Optional[str] = None - - class ClipGradAndStepResponse(OkResponse): """Response for /clip_grad_and_step endpoint.""" pass diff --git a/tests/server/config/test_server_config.py b/tests/server/config/test_server_config.py index bebd63c9..433f85a5 100644 --- a/tests/server/config/test_server_config.py +++ b/tests/server/config/test_server_config.py @@ -24,10 +24,6 @@ _PERSISTENCE_VARIANTS = st.one_of( st.fixed_dictionaries({'mode': st.just('memory')}), - st.fixed_dictionaries({ - 'mode': st.just('file'), - 'file_path': st.just('/tmp/state.json') - }), st.fixed_dictionaries({ 'mode': st.just('redis'), 'redis_url': st.just('redis://localhost:6379/0') @@ -86,13 +82,6 @@ def test_redis_mode_missing_url() -> None: assert 'persistence.redis_url' in msg or 'redis_url' in msg -def test_file_mode_missing_path() -> None: - with pytest.raises(ValidationError) as exc: - ServerConfig.model_validate({'persistence': {'mode': 'file'}}) - msg = str(exc.value) - assert 'persistence.file_path' in msg or 'file_path' in msg - - @settings(max_examples=100) @given(bad_backend=st.text(min_size=1, max_size=8).filter(lambda s: s not in ('mock', 'transformers', 'megatron'))) def test_bad_backend_names_field(bad_backend: str) -> None: diff --git a/tests/server/contract/client_api_baseline.json b/tests/server/contract/client_api_baseline.json deleted file mode 100644 index 44672f77..00000000 --- a/tests/server/contract/client_api_baseline.json +++ /dev/null @@ -1,8353 +0,0 @@ -{ - "data_plane": { - "paths": { - "/twinkle/append": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "DataRef": { - "description": "Opaque reference to rows stored in the server-side TransferQueue.", - "properties": { - "fields": { - "items": { - "type": "string" - }, - "title": "Fields", - "type": "array" - }, - "kind": { - "default": "data", - "title": "Kind", - "type": "string" - }, - "num_tokens": { - "default": 0, - "title": "Num Tokens", - "type": "integer" - }, - "ref_id": { - "title": "Ref Id", - "type": "string" - }, - "size": { - "title": "Size", - "type": "integer" - } - }, - "required": [ - "ref_id", - "size" - ], - "title": "DataRef", - "type": "object" - } - }, - "properties": { - "ref": { - "$ref": "#/$defs/DataRef" - }, - "rows": { - "items": { - "additionalProperties": true, - "type": "object" - }, - "title": "Rows", - "type": "array" - }, - "tags": { - "anyOf": [ - { - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Tags" - } - }, - "required": [ - "ref", - "rows" - ], - "title": "DataAppendRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "append", - "path": [], - "query": [], - "response": { - "description": "Opaque reference to rows stored in the server-side TransferQueue.", - "properties": { - "fields": { - "items": { - "type": "string" - }, - "title": "Fields", - "type": "array" - }, - "kind": { - "default": "data", - "title": "Kind", - "type": "string" - }, - "num_tokens": { - "default": 0, - "title": "Num Tokens", - "type": "integer" - }, - "ref_id": { - "title": "Ref Id", - "type": "string" - }, - "size": { - "title": "Size", - "type": "integer" - } - }, - "required": [ - "ref_id", - "size" - ], - "title": "DataRef", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/get": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "DataRef": { - "description": "Opaque reference to rows stored in the server-side TransferQueue.", - "properties": { - "fields": { - "items": { - "type": "string" - }, - "title": "Fields", - "type": "array" - }, - "kind": { - "default": "data", - "title": "Kind", - "type": "string" - }, - "num_tokens": { - "default": 0, - "title": "Num Tokens", - "type": "integer" - }, - "ref_id": { - "title": "Ref Id", - "type": "string" - }, - "size": { - "title": "Size", - "type": "integer" - } - }, - "required": [ - "ref_id", - "size" - ], - "title": "DataRef", - "type": "object" - } - }, - "properties": { - "fields": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Fields" - }, - "include_tags": { - "default": false, - "title": "Include Tags", - "type": "boolean" - }, - "ref": { - "$ref": "#/$defs/DataRef" - } - }, - "required": [ - "ref" - ], - "title": "DataGetRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "get", - "path": [], - "query": [], - "response": { - "properties": { - "rows": { - "items": { - "additionalProperties": true, - "type": "object" - }, - "title": "Rows", - "type": "array" - }, - "tags": { - "items": { - "additionalProperties": true, - "type": "object" - }, - "title": "Tags", - "type": "array" - } - }, - "required": [ - "rows" - ], - "title": "DataRowsResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/put": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "properties": { - "kind": { - "default": "data", - "title": "Kind", - "type": "string" - }, - "rows": { - "items": { - "additionalProperties": true, - "type": "object" - }, - "title": "Rows", - "type": "array" - }, - "tags": { - "anyOf": [ - { - "items": { - "additionalProperties": true, - "type": "object" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Tags" - } - }, - "required": [ - "rows" - ], - "title": "DataPutRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "put", - "path": [], - "query": [], - "response": { - "description": "Opaque reference to rows stored in the server-side TransferQueue.", - "properties": { - "fields": { - "items": { - "type": "string" - }, - "title": "Fields", - "type": "array" - }, - "kind": { - "default": "data", - "title": "Kind", - "type": "string" - }, - "num_tokens": { - "default": 0, - "title": "Num Tokens", - "type": "integer" - }, - "ref_id": { - "title": "Ref Id", - "type": "string" - }, - "size": { - "title": "Size", - "type": "integer" - } - }, - "required": [ - "ref_id", - "size" - ], - "title": "DataRef", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/release": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "DataRef": { - "description": "Opaque reference to rows stored in the server-side TransferQueue.", - "properties": { - "fields": { - "items": { - "type": "string" - }, - "title": "Fields", - "type": "array" - }, - "kind": { - "default": "data", - "title": "Kind", - "type": "string" - }, - "num_tokens": { - "default": 0, - "title": "Num Tokens", - "type": "integer" - }, - "ref_id": { - "title": "Ref Id", - "type": "string" - }, - "size": { - "title": "Size", - "type": "integer" - } - }, - "required": [ - "ref_id", - "size" - ], - "title": "DataRef", - "type": "object" - } - }, - "properties": { - "ref": { - "$ref": "#/$defs/DataRef" - } - }, - "required": [ - "ref" - ], - "title": "DataReleaseRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "release", - "path": [], - "query": [], - "response": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - } - } - }, - "gateway": { - "paths": { - "/asample": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "EncodedTextChunk": { - "additionalProperties": false, - "properties": { - "tokens": { - "items": { - "type": "integer" - }, - "title": "Tokens", - "type": "array" - }, - "type": { - "const": "encoded_text", - "default": "encoded_text", - "title": "Type", - "type": "string" - } - }, - "required": [ - "tokens" - ], - "title": "EncodedTextChunk", - "type": "object" - }, - "ImageAssetPointerChunk": { - "additionalProperties": false, - "properties": { - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "type": { - "const": "image_asset_pointer", - "default": "image_asset_pointer", - "title": "Type", - "type": "string" - } - }, - "required": [ - "format", - "location" - ], - "title": "ImageAssetPointerChunk", - "type": "object" - }, - "ImageChunk": { - "additionalProperties": false, - "properties": { - "data": { - "format": "binary", - "title": "Data", - "type": "string" - }, - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "type": { - "const": "image", - "default": "image", - "title": "Type", - "type": "string" - } - }, - "required": [ - "data", - "format" - ], - "title": "ImageChunk", - "type": "object" - }, - "ModelInput": { - "additionalProperties": false, - "properties": { - "chunks": { - "items": { - "anyOf": [ - { - "$ref": "#/$defs/EncodedTextChunk" - }, - { - "$ref": "#/$defs/ImageAssetPointerChunk" - }, - { - "$ref": "#/$defs/ImageChunk" - } - ] - }, - "title": "Chunks", - "type": "array" - } - }, - "required": [ - "chunks" - ], - "title": "ModelInput", - "type": "object" - }, - "SamplingParams": { - "properties": { - "max_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Max Tokens" - }, - "seed": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seed" - }, - "stop": { - "anyOf": [ - { - "type": "string" - }, - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Stop" - }, - "temperature": { - "default": 1, - "title": "Temperature", - "type": "number" - }, - "top_k": { - "default": -1, - "title": "Top K", - "type": "integer" - }, - "top_p": { - "default": 1, - "title": "Top P", - "type": "number" - } - }, - "title": "SamplingParams", - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "base_model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Base Model" - }, - "model_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Path" - }, - "num_samples": { - "default": 1, - "title": "Num Samples", - "type": "integer" - }, - "prompt": { - "$ref": "#/$defs/ModelInput" - }, - "prompt_logprobs": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Prompt Logprobs" - }, - "sampling_params": { - "$ref": "#/$defs/SamplingParams" - }, - "sampling_session_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sampling Session Id" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - }, - "topk_prompt_logprobs": { - "default": 0, - "title": "Topk Prompt Logprobs", - "type": "integer" - }, - "type": { - "const": "sample", - "default": "sample", - "title": "Type", - "type": "string" - } - }, - "required": [ - "prompt", - "sampling_params" - ], - "title": "SampleRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "asample", - "path": [], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/chat/completions": { - "POST": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "chat_completions", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/create_model": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "LoraConfig": { - "additionalProperties": false, - "properties": { - "rank": { - "title": "Rank", - "type": "integer" - }, - "seed": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seed" - }, - "train_attn": { - "default": true, - "title": "Train Attn", - "type": "boolean" - }, - "train_mlp": { - "default": true, - "title": "Train Mlp", - "type": "boolean" - }, - "train_unembed": { - "default": true, - "title": "Train Unembed", - "type": "boolean" - } - }, - "required": [ - "rank" - ], - "title": "LoraConfig", - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "base_model": { - "title": "Base Model", - "type": "string" - }, - "lora_config": { - "anyOf": [ - { - "$ref": "#/$defs/LoraConfig" - }, - { - "type": "null" - } - ], - "default": null - }, - "model_seq_id": { - "title": "Model Seq Id", - "type": "integer" - }, - "session_id": { - "title": "Session Id", - "type": "string" - }, - "type": { - "const": "create_model", - "default": "create_model", - "title": "Type", - "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" - } - }, - "required": [ - "session_id", - "model_seq_id", - "base_model" - ], - "title": "CreateModelRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "create_model", - "path": [], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/create_sampling_session": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "base_model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Base Model" - }, - "model_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Path" - }, - "sampling_session_seq_id": { - "title": "Sampling Session Seq Id", - "type": "integer" - }, - "session_id": { - "title": "Session Id", - "type": "string" - }, - "type": { - "const": "create_sampling_session", - "default": "create_sampling_session", - "title": "Type", - "type": "string" - } - }, - "required": [ - "session_id", - "sampling_session_seq_id" - ], - "title": "CreateSamplingSessionRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "create_sampling_session", - "path": [], - "query": [], - "response": { - "properties": { - "sampling_session_id": { - "title": "Sampling Session Id", - "type": "string" - }, - "type": { - "const": "create_sampling_session", - "default": "create_sampling_session", - "title": "Type", - "type": "string" - } - }, - "required": [ - "sampling_session_id" - ], - "title": "CreateSamplingSessionResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/create_session": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "project_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Project Id" - }, - "sdk_version": { - "title": "Sdk Version", - "type": "string" - }, - "tags": { - "items": { - "type": "string" - }, - "title": "Tags", - "type": "array" - }, - "type": { - "const": "create_session", - "default": "create_session", - "title": "Type", - "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "title": "User Metadata" - } - }, - "required": [ - "tags", - "user_metadata", - "sdk_version" - ], - "title": "CreateSessionRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "create_session", - "path": [], - "query": [], - "response": { - "properties": { - "error_message": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Error Message" - }, - "info_message": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Info Message" - }, - "session_id": { - "title": "Session Id", - "type": "string" - }, - "type": { - "const": "create_session", - "default": "create_session", - "title": "Type", - "type": "string" - }, - "warning_message": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Warning Message" - } - }, - "required": [ - "session_id" - ], - "title": "CreateSessionResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/forward": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "Datum": { - "additionalProperties": false, - "properties": { - "loss_fn_inputs": { - "additionalProperties": { - "$ref": "#/$defs/TensorData" - }, - "title": "Loss Fn Inputs", - "type": "object" - }, - "model_input": { - "$ref": "#/$defs/ModelInput" - } - }, - "required": [ - "loss_fn_inputs", - "model_input" - ], - "title": "Datum", - "type": "object" - }, - "EncodedTextChunk": { - "additionalProperties": false, - "properties": { - "tokens": { - "items": { - "type": "integer" - }, - "title": "Tokens", - "type": "array" - }, - "type": { - "const": "encoded_text", - "default": "encoded_text", - "title": "Type", - "type": "string" - } - }, - "required": [ - "tokens" - ], - "title": "EncodedTextChunk", - "type": "object" - }, - "ForwardBackwardInput": { - "additionalProperties": false, - "properties": { - "data": { - "items": { - "$ref": "#/$defs/Datum" - }, - "title": "Data", - "type": "array" - }, - "loss_fn": { - "enum": [ - "cross_entropy", - "importance_sampling", - "ppo", - "cispo", - "dro" - ], - "title": "Loss Fn", - "type": "string" - }, - "loss_fn_config": { - "anyOf": [ - { - "additionalProperties": { - "type": "number" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Loss Fn Config" - } - }, - "required": [ - "data", - "loss_fn" - ], - "title": "ForwardBackwardInput", - "type": "object" - }, - "ImageAssetPointerChunk": { - "additionalProperties": false, - "properties": { - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "type": { - "const": "image_asset_pointer", - "default": "image_asset_pointer", - "title": "Type", - "type": "string" - } - }, - "required": [ - "format", - "location" - ], - "title": "ImageAssetPointerChunk", - "type": "object" - }, - "ImageChunk": { - "additionalProperties": false, - "properties": { - "data": { - "format": "binary", - "title": "Data", - "type": "string" - }, - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "type": { - "const": "image", - "default": "image", - "title": "Type", - "type": "string" - } - }, - "required": [ - "data", - "format" - ], - "title": "ImageChunk", - "type": "object" - }, - "ModelInput": { - "additionalProperties": false, - "properties": { - "chunks": { - "items": { - "anyOf": [ - { - "$ref": "#/$defs/EncodedTextChunk" - }, - { - "$ref": "#/$defs/ImageAssetPointerChunk" - }, - { - "$ref": "#/$defs/ImageChunk" - } - ] - }, - "title": "Chunks", - "type": "array" - } - }, - "required": [ - "chunks" - ], - "title": "ModelInput", - "type": "object" - }, - "TensorData": { - "additionalProperties": false, - "properties": { - "data": { - "anyOf": [ - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "items": { - "type": "number" - }, - "type": "array" - } - ], - "title": "Data" - }, - "dtype": { - "enum": [ - "int64", - "float32" - ], - "title": "Dtype", - "type": "string" - }, - "shape": { - "anyOf": [ - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Shape" - } - }, - "required": [ - "data", - "dtype" - ], - "title": "TensorData", - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "forward_input": { - "$ref": "#/$defs/ForwardBackwardInput" - }, - "model_id": { - "title": "Model Id", - "type": "string" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - } - }, - "required": [ - "forward_input", - "model_id" - ], - "title": "ForwardRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "forward", - "path": [], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/forward_backward": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "Datum": { - "additionalProperties": false, - "properties": { - "loss_fn_inputs": { - "additionalProperties": { - "$ref": "#/$defs/TensorData" - }, - "title": "Loss Fn Inputs", - "type": "object" - }, - "model_input": { - "$ref": "#/$defs/ModelInput" - } - }, - "required": [ - "loss_fn_inputs", - "model_input" - ], - "title": "Datum", - "type": "object" - }, - "EncodedTextChunk": { - "additionalProperties": false, - "properties": { - "tokens": { - "items": { - "type": "integer" - }, - "title": "Tokens", - "type": "array" - }, - "type": { - "const": "encoded_text", - "default": "encoded_text", - "title": "Type", - "type": "string" - } - }, - "required": [ - "tokens" - ], - "title": "EncodedTextChunk", - "type": "object" - }, - "ForwardBackwardInput": { - "additionalProperties": false, - "properties": { - "data": { - "items": { - "$ref": "#/$defs/Datum" - }, - "title": "Data", - "type": "array" - }, - "loss_fn": { - "enum": [ - "cross_entropy", - "importance_sampling", - "ppo", - "cispo", - "dro" - ], - "title": "Loss Fn", - "type": "string" - }, - "loss_fn_config": { - "anyOf": [ - { - "additionalProperties": { - "type": "number" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Loss Fn Config" - } - }, - "required": [ - "data", - "loss_fn" - ], - "title": "ForwardBackwardInput", - "type": "object" - }, - "ImageAssetPointerChunk": { - "additionalProperties": false, - "properties": { - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "type": { - "const": "image_asset_pointer", - "default": "image_asset_pointer", - "title": "Type", - "type": "string" - } - }, - "required": [ - "format", - "location" - ], - "title": "ImageAssetPointerChunk", - "type": "object" - }, - "ImageChunk": { - "additionalProperties": false, - "properties": { - "data": { - "format": "binary", - "title": "Data", - "type": "string" - }, - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "type": { - "const": "image", - "default": "image", - "title": "Type", - "type": "string" - } - }, - "required": [ - "data", - "format" - ], - "title": "ImageChunk", - "type": "object" - }, - "ModelInput": { - "additionalProperties": false, - "properties": { - "chunks": { - "items": { - "anyOf": [ - { - "$ref": "#/$defs/EncodedTextChunk" - }, - { - "$ref": "#/$defs/ImageAssetPointerChunk" - }, - { - "$ref": "#/$defs/ImageChunk" - } - ] - }, - "title": "Chunks", - "type": "array" - } - }, - "required": [ - "chunks" - ], - "title": "ModelInput", - "type": "object" - }, - "TensorData": { - "additionalProperties": false, - "properties": { - "data": { - "anyOf": [ - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "items": { - "type": "number" - }, - "type": "array" - } - ], - "title": "Data" - }, - "dtype": { - "enum": [ - "int64", - "float32" - ], - "title": "Dtype", - "type": "string" - }, - "shape": { - "anyOf": [ - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Shape" - } - }, - "required": [ - "data", - "dtype" - ], - "title": "TensorData", - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "forward_backward_input": { - "$ref": "#/$defs/ForwardBackwardInput" - }, - "model_id": { - "title": "Model Id", - "type": "string" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - } - }, - "required": [ - "forward_backward_input", - "model_id" - ], - "title": "ForwardBackwardRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "forward_backward", - "path": [], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/get_info": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "model_id": { - "title": "Model Id", - "type": "string" - }, - "type": { - "const": "get_info", - "default": "get_info", - "title": "Type", - "type": "string" - } - }, - "required": [ - "model_id" - ], - "title": "GetInfoRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "get_info", - "path": [], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/get_server_capabilities": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "get_server_capabilities", - "path": [], - "query": [], - "response": { - "$defs": { - "SupportedModel": { - "description": "Information about a model supported by the server.", - "properties": { - "model_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Name" - } - }, - "title": "SupportedModel", - "type": "object" - } - }, - "description": "Response containing the server's supported models and capabilities.", - "properties": { - "supported_models": { - "items": { - "$ref": "#/$defs/SupportedModel" - }, - "title": "Supported Models", - "type": "array" - } - }, - "required": [ - "supported_models" - ], - "title": "GetServerCapabilitiesResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/healthz": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "healthz", - "path": [], - "query": [], - "response": { - "properties": { - "status": { - "const": "ok", - "title": "Status", - "type": "string" - } - }, - "required": [ - "status" - ], - "title": "HealthResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/load_weights": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "model_id": { - "title": "Model Id", - "type": "string" - }, - "optimizer": { - "title": "Optimizer", - "type": "boolean" - }, - "path": { - "title": "Path", - "type": "string" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - }, - "type": { - "const": "load_weights", - "default": "load_weights", - "title": "Type", - "type": "string" - } - }, - "required": [ - "model_id", - "path", - "optimizer" - ], - "title": "LoadWeightsRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "load_weights", - "path": [], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/models": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "list_models", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/optim_step": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "AdamParams": { - "additionalProperties": false, - "properties": { - "beta1": { - "default": 0.9, - "title": "Beta1", - "type": "number" - }, - "beta2": { - "default": 0.95, - "title": "Beta2", - "type": "number" - }, - "eps": { - "default": 1e-12, - "title": "Eps", - "type": "number" - }, - "grad_clip_norm": { - "default": 0.0, - "title": "Grad Clip Norm", - "type": "number" - }, - "learning_rate": { - "default": 0.0001, - "title": "Learning Rate", - "type": "number" - }, - "weight_decay": { - "default": 0.0, - "title": "Weight Decay", - "type": "number" - } - }, - "title": "AdamParams", - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "adam_params": { - "$ref": "#/$defs/AdamParams" - }, - "model_id": { - "title": "Model Id", - "type": "string" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - }, - "type": { - "const": "optim_step", - "default": "optim_step", - "title": "Type", - "type": "string" - } - }, - "required": [ - "adam_params", - "model_id" - ], - "title": "OptimStepRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "optim_step", - "path": [], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/retrieve_future": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "allow_metadata_only": { - "default": false, - "title": "Allow Metadata Only", - "type": "boolean" - }, - "request_id": { - "title": "Request Id", - "type": "string" - } - }, - "required": [ - "request_id" - ], - "title": "FutureRetrieveRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "retrieve_future", - "path": [], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/save_weights": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "model_id": { - "title": "Model Id", - "type": "string" - }, - "path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Path" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - }, - "ttl_seconds": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Ttl Seconds" - }, - "type": { - "const": "save_weights", - "default": "save_weights", - "title": "Type", - "type": "string" - } - }, - "required": [ - "model_id" - ], - "title": "SaveWeightsRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "save_weights", - "path": [], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/save_weights_for_sampler": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "model_id": { - "title": "Model Id", - "type": "string" - }, - "path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Path" - }, - "sampling_session_seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sampling Session Seq Id" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - }, - "ttl_seconds": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Ttl Seconds" - }, - "type": { - "const": "save_weights_for_sampler", - "default": "save_weights_for_sampler", - "title": "Type", - "type": "string" - } - }, - "required": [ - "model_id" - ], - "title": "SaveWeightsForSamplerRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "save_weights_for_sampler", - "path": [], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/session_heartbeat": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "session_id": { - "title": "Session Id", - "type": "string" - }, - "type": { - "const": "session_heartbeat", - "default": "session_heartbeat", - "title": "Type", - "type": "string" - } - }, - "required": [ - "session_id" - ], - "title": "SessionHeartbeatRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "session_heartbeat", - "path": [], - "query": [], - "response": { - "properties": { - "type": { - "const": "session_heartbeat", - "default": "session_heartbeat", - "title": "Type", - "type": "string" - } - }, - "title": "SessionHeartbeatResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/telemetry": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "GenericEvent": { - "properties": { - "event": { - "enum": [ - "SESSION_START", - "SESSION_END", - "UNHANDLED_EXCEPTION", - "GENERIC_EVENT" - ], - "title": "Event", - "type": "string" - }, - "event_data": { - "additionalProperties": true, - "default": {}, - "title": "Event Data", - "type": "object" - }, - "event_id": { - "title": "Event Id", - "type": "string" - }, - "event_name": { - "title": "Event Name", - "type": "string" - }, - "event_session_index": { - "title": "Event Session Index", - "type": "integer" - }, - "severity": { - "enum": [ - "DEBUG", - "INFO", - "WARNING", - "ERROR", - "CRITICAL" - ], - "title": "Severity", - "type": "string" - }, - "timestamp": { - "format": "date-time", - "title": "Timestamp", - "type": "string" - } - }, - "required": [ - "event", - "event_id", - "event_name", - "event_session_index", - "severity", - "timestamp" - ], - "title": "GenericEvent", - "type": "object" - }, - "SessionEndEvent": { - "properties": { - "duration": { - "title": "Duration", - "type": "string" - }, - "event": { - "enum": [ - "SESSION_START", - "SESSION_END", - "UNHANDLED_EXCEPTION", - "GENERIC_EVENT" - ], - "title": "Event", - "type": "string" - }, - "event_id": { - "title": "Event Id", - "type": "string" - }, - "event_session_index": { - "title": "Event Session Index", - "type": "integer" - }, - "severity": { - "enum": [ - "DEBUG", - "INFO", - "WARNING", - "ERROR", - "CRITICAL" - ], - "title": "Severity", - "type": "string" - }, - "timestamp": { - "format": "date-time", - "title": "Timestamp", - "type": "string" - } - }, - "required": [ - "duration", - "event", - "event_id", - "event_session_index", - "severity", - "timestamp" - ], - "title": "SessionEndEvent", - "type": "object" - }, - "SessionStartEvent": { - "properties": { - "event": { - "enum": [ - "SESSION_START", - "SESSION_END", - "UNHANDLED_EXCEPTION", - "GENERIC_EVENT" - ], - "title": "Event", - "type": "string" - }, - "event_id": { - "title": "Event Id", - "type": "string" - }, - "event_session_index": { - "title": "Event Session Index", - "type": "integer" - }, - "severity": { - "enum": [ - "DEBUG", - "INFO", - "WARNING", - "ERROR", - "CRITICAL" - ], - "title": "Severity", - "type": "string" - }, - "timestamp": { - "format": "date-time", - "title": "Timestamp", - "type": "string" - } - }, - "required": [ - "event", - "event_id", - "event_session_index", - "severity", - "timestamp" - ], - "title": "SessionStartEvent", - "type": "object" - }, - "UnhandledExceptionEvent": { - "properties": { - "error_message": { - "title": "Error Message", - "type": "string" - }, - "error_type": { - "title": "Error Type", - "type": "string" - }, - "event": { - "enum": [ - "SESSION_START", - "SESSION_END", - "UNHANDLED_EXCEPTION", - "GENERIC_EVENT" - ], - "title": "Event", - "type": "string" - }, - "event_id": { - "title": "Event Id", - "type": "string" - }, - "event_session_index": { - "title": "Event Session Index", - "type": "integer" - }, - "severity": { - "enum": [ - "DEBUG", - "INFO", - "WARNING", - "ERROR", - "CRITICAL" - ], - "title": "Severity", - "type": "string" - }, - "timestamp": { - "format": "date-time", - "title": "Timestamp", - "type": "string" - }, - "traceback": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Traceback" - } - }, - "required": [ - "error_message", - "error_type", - "event", - "event_id", - "event_session_index", - "severity", - "timestamp" - ], - "title": "UnhandledExceptionEvent", - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "events": { - "items": { - "anyOf": [ - { - "$ref": "#/$defs/SessionStartEvent" - }, - { - "$ref": "#/$defs/SessionEndEvent" - }, - { - "$ref": "#/$defs/UnhandledExceptionEvent" - }, - { - "$ref": "#/$defs/GenericEvent" - } - ] - }, - "title": "Events", - "type": "array" - }, - "platform": { - "title": "Platform", - "type": "string" - }, - "sdk_version": { - "title": "Sdk Version", - "type": "string" - }, - "session_id": { - "title": "Session Id", - "type": "string" - } - }, - "required": [ - "events", - "platform", - "sdk_version", - "session_id" - ], - "title": "TelemetrySendRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "telemetry", - "path": [], - "query": [], - "response": { - "properties": { - "status": { - "const": "accepted", - "title": "Status", - "type": "string" - } - }, - "required": [ - "status" - ], - "title": "TelemetryResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/training_runs": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "get_training_runs", - "path": [], - "query": [ - { - "name": "limit", - "required": false, - "schema": { - "type": "integer" - } - }, - { - "name": "offset", - "required": false, - "schema": { - "type": "integer" - } - } - ], - "response": { - "$defs": { - "Checkpoint": { - "properties": { - "checkpoint_id": { - "title": "Checkpoint Id", - "type": "string" - }, - "checkpoint_type": { - "enum": [ - "training", - "sampler" - ], - "title": "Checkpoint Type", - "type": "string" - }, - "expires_at": { - "anyOf": [ - { - "format": "date-time", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expires At" - }, - "public": { - "default": false, - "title": "Public", - "type": "boolean" - }, - "size_bytes": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Size Bytes" - }, - "time": { - "format": "date-time", - "title": "Time", - "type": "string" - }, - "tinker_path": { - "title": "Tinker Path", - "type": "string" - } - }, - "required": [ - "checkpoint_id", - "checkpoint_type", - "time", - "tinker_path" - ], - "title": "Checkpoint", - "type": "object" - }, - "Cursor": { - "properties": { - "limit": { - "title": "Limit", - "type": "integer" - }, - "offset": { - "title": "Offset", - "type": "integer" - }, - "total_count": { - "title": "Total Count", - "type": "integer" - } - }, - "required": [ - "offset", - "limit", - "total_count" - ], - "title": "Cursor", - "type": "object" - }, - "TrainingRun": { - "properties": { - "base_model": { - "title": "Base Model", - "type": "string" - }, - "corrupted": { - "default": false, - "title": "Corrupted", - "type": "boolean" - }, - "is_lora": { - "title": "Is Lora", - "type": "boolean" - }, - "last_checkpoint": { - "anyOf": [ - { - "$ref": "#/$defs/Checkpoint" - }, - { - "type": "null" - } - ], - "default": null - }, - "last_request_time": { - "format": "date-time", - "title": "Last Request Time", - "type": "string" - }, - "last_sampler_checkpoint": { - "anyOf": [ - { - "$ref": "#/$defs/Checkpoint" - }, - { - "type": "null" - } - ], - "default": null - }, - "lora_rank": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Lora Rank" - }, - "model_owner": { - "title": "Model Owner", - "type": "string" - }, - "training_run_id": { - "title": "Training Run Id", - "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" - } - }, - "required": [ - "training_run_id", - "base_model", - "model_owner", - "is_lora", - "last_request_time" - ], - "title": "TrainingRun", - "type": "object" - } - }, - "properties": { - "cursor": { - "$ref": "#/$defs/Cursor" - }, - "training_runs": { - "items": { - "$ref": "#/$defs/TrainingRun" - }, - "title": "Training Runs", - "type": "array" - } - }, - "required": [ - "training_runs", - "cursor" - ], - "title": "TrainingRunsResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/training_runs/{run_id}": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "get_training_run", - "path": [ - { - "name": "run_id", - "required": true, - "schema": { - "type": "string" - } - } - ], - "query": [], - "response": { - "$defs": { - "Checkpoint": { - "properties": { - "checkpoint_id": { - "title": "Checkpoint Id", - "type": "string" - }, - "checkpoint_type": { - "enum": [ - "training", - "sampler" - ], - "title": "Checkpoint Type", - "type": "string" - }, - "expires_at": { - "anyOf": [ - { - "format": "date-time", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expires At" - }, - "public": { - "default": false, - "title": "Public", - "type": "boolean" - }, - "size_bytes": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Size Bytes" - }, - "time": { - "format": "date-time", - "title": "Time", - "type": "string" - }, - "tinker_path": { - "title": "Tinker Path", - "type": "string" - } - }, - "required": [ - "checkpoint_id", - "checkpoint_type", - "time", - "tinker_path" - ], - "title": "Checkpoint", - "type": "object" - } - }, - "properties": { - "base_model": { - "title": "Base Model", - "type": "string" - }, - "corrupted": { - "default": false, - "title": "Corrupted", - "type": "boolean" - }, - "is_lora": { - "title": "Is Lora", - "type": "boolean" - }, - "last_checkpoint": { - "anyOf": [ - { - "$ref": "#/$defs/Checkpoint" - }, - { - "type": "null" - } - ], - "default": null - }, - "last_request_time": { - "format": "date-time", - "title": "Last Request Time", - "type": "string" - }, - "last_sampler_checkpoint": { - "anyOf": [ - { - "$ref": "#/$defs/Checkpoint" - }, - { - "type": "null" - } - ], - "default": null - }, - "lora_rank": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Lora Rank" - }, - "model_owner": { - "title": "Model Owner", - "type": "string" - }, - "training_run_id": { - "title": "Training Run Id", - "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" - } - }, - "required": [ - "training_run_id", - "base_model", - "model_owner", - "is_lora", - "last_request_time" - ], - "title": "TrainingRun", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/training_runs/{run_id}/checkpoints": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "get_run_checkpoints", - "path": [ - { - "name": "run_id", - "required": true, - "schema": { - "type": "string" - } - } - ], - "query": [], - "response": { - "$defs": { - "Checkpoint": { - "properties": { - "checkpoint_id": { - "title": "Checkpoint Id", - "type": "string" - }, - "checkpoint_type": { - "enum": [ - "training", - "sampler" - ], - "title": "Checkpoint Type", - "type": "string" - }, - "expires_at": { - "anyOf": [ - { - "format": "date-time", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expires At" - }, - "public": { - "default": false, - "title": "Public", - "type": "boolean" - }, - "size_bytes": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Size Bytes" - }, - "time": { - "format": "date-time", - "title": "Time", - "type": "string" - }, - "tinker_path": { - "title": "Tinker Path", - "type": "string" - } - }, - "required": [ - "checkpoint_id", - "checkpoint_type", - "time", - "tinker_path" - ], - "title": "Checkpoint", - "type": "object" - }, - "Cursor": { - "properties": { - "limit": { - "title": "Limit", - "type": "integer" - }, - "offset": { - "title": "Offset", - "type": "integer" - }, - "total_count": { - "title": "Total Count", - "type": "integer" - } - }, - "required": [ - "offset", - "limit", - "total_count" - ], - "title": "Cursor", - "type": "object" - } - }, - "properties": { - "checkpoints": { - "items": { - "$ref": "#/$defs/Checkpoint" - }, - "title": "Checkpoints", - "type": "array" - }, - "cursor": { - "anyOf": [ - { - "$ref": "#/$defs/Cursor" - }, - { - "type": "null" - } - ], - "default": null - } - }, - "required": [ - "checkpoints" - ], - "title": "CheckpointsListResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/training_runs/{run_id}/checkpoints/{checkpoint_id}": { - "DELETE": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "delete_run_checkpoint", - "path": [ - { - "name": "run_id", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "checkpoint_id", - "required": true, - "schema": { - "type": "string" - } - } - ], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/training_runs/{run_id}/checkpoints/{checkpoint_id}/publish": { - "POST": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "publish_checkpoint", - "path": [ - { - "name": "run_id", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "checkpoint_id", - "required": true, - "schema": { - "type": "string" - } - } - ], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/capacity_info": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "get_capacity_info", - "path": [], - "query": [], - "response": { - "description": "Response body for the /capacity_info endpoint.", - "properties": { - "free_loras": { - "title": "Free Loras", - "type": "integer" - }, - "max_loras": { - "title": "Max Loras", - "type": "integer" - }, - "used_loras": { - "title": "Used Loras", - "type": "integer" - } - }, - "required": [ - "max_loras", - "used_loras", - "free_loras" - ], - "title": "CapacityInfoResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/checkpoint_path/{run_id}/{checkpoint_id}": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "get_checkpoint_path", - "path": [ - { - "name": "run_id", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "checkpoint_id", - "required": true, - "schema": { - "type": "string" - } - } - ], - "query": [], - "response": { - "description": "Response body for the /checkpoint_path endpoint.", - "properties": { - "path": { - "title": "Path", - "type": "string" - }, - "twinkle_path": { - "title": "Twinkle Path", - "type": "string" - } - }, - "required": [ - "path", - "twinkle_path" - ], - "title": "CheckpointPathResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/create_session": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "description": "Request body for POST /twinkle/create_session.", - "properties": { - "metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Metadata" - } - }, - "title": "CreateSessionRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "create_session", - "path": [], - "query": [], - "response": { - "description": "Response body for POST /twinkle/create_session.", - "properties": { - "session_id": { - "title": "Session Id", - "type": "string" - } - }, - "required": [ - "session_id" - ], - "title": "CreateSessionResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/get_server_capabilities": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "get_server_capabilities", - "path": [], - "query": [], - "response": { - "$defs": { - "SupportedModel": { - "description": "Information about a supported model.", - "properties": { - "model_name": { - "title": "Model Name", - "type": "string" - } - }, - "required": [ - "model_name" - ], - "title": "SupportedModel", - "type": "object" - } - }, - "description": "Response body for the /get_server_capabilities endpoint.", - "properties": { - "supported_models": { - "items": { - "$ref": "#/$defs/SupportedModel" - }, - "title": "Supported Models", - "type": "array" - } - }, - "required": [ - "supported_models" - ], - "title": "GetServerCapabilitiesResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/healthz": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "healthz", - "path": [], - "query": [], - "response": { - "properties": { - "status": { - "title": "Status", - "type": "string" - } - }, - "required": [ - "status" - ], - "title": "HealthResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/healthz/deep": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "healthz_deep", - "path": [], - "query": [], - "response": { - "pythonType": "dict" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/session_heartbeat": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "description": "Request body for POST /twinkle/session_heartbeat.", - "properties": { - "session_id": { - "title": "Session Id", - "type": "string" - } - }, - "required": [ - "session_id" - ], - "title": "SessionHeartbeatRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "session_heartbeat", - "path": [], - "query": [], - "response": { - "description": "Response body for POST /twinkle/session_heartbeat.", - "properties": {}, - "title": "SessionHeartbeatResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/status": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "status", - "path": [], - "query": [], - "response": { - "pythonType": "dict" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/training_runs": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "get_training_runs", - "path": [], - "query": [ - { - "name": "limit", - "required": false, - "schema": { - "type": "integer" - } - }, - { - "name": "offset", - "required": false, - "schema": { - "type": "integer" - } - } - ], - "response": { - "$defs": { - "Cursor": { - "properties": { - "limit": { - "title": "Limit", - "type": "integer" - }, - "offset": { - "title": "Offset", - "type": "integer" - }, - "total_count": { - "title": "Total Count", - "type": "integer" - } - }, - "required": [ - "limit", - "offset", - "total_count" - ], - "title": "Cursor", - "type": "object" - }, - "TrainingRun": { - "description": "Twinkle training run model.", - "properties": { - "base_model": { - "title": "Base Model", - "type": "string" - }, - "corrupted": { - "default": false, - "title": "Corrupted", - "type": "boolean" - }, - "is_lora": { - "default": false, - "title": "Is Lora", - "type": "boolean" - }, - "last_checkpoint": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Last Checkpoint" - }, - "last_request_time": { - "anyOf": [ - { - "format": "date-time", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Last Request Time" - }, - "last_sampler_checkpoint": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Last Sampler Checkpoint" - }, - "lora_rank": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Lora Rank" - }, - "model_owner": { - "title": "Model Owner", - "type": "string" - }, - "save_dir": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Save Dir" - }, - "training_run_id": { - "title": "Training Run Id", - "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" - } - }, - "required": [ - "training_run_id", - "base_model", - "model_owner" - ], - "title": "TrainingRun", - "type": "object" - } - }, - "properties": { - "cursor": { - "$ref": "#/$defs/Cursor" - }, - "training_runs": { - "items": { - "$ref": "#/$defs/TrainingRun" - }, - "title": "Training Runs", - "type": "array" - } - }, - "required": [ - "training_runs", - "cursor" - ], - "title": "TrainingRunsResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/training_runs/{run_id}": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "get_training_run", - "path": [ - { - "name": "run_id", - "required": true, - "schema": { - "type": "string" - } - } - ], - "query": [], - "response": { - "description": "Twinkle training run model.", - "properties": { - "base_model": { - "title": "Base Model", - "type": "string" - }, - "corrupted": { - "default": false, - "title": "Corrupted", - "type": "boolean" - }, - "is_lora": { - "default": false, - "title": "Is Lora", - "type": "boolean" - }, - "last_checkpoint": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Last Checkpoint" - }, - "last_request_time": { - "anyOf": [ - { - "format": "date-time", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Last Request Time" - }, - "last_sampler_checkpoint": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Last Sampler Checkpoint" - }, - "lora_rank": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Lora Rank" - }, - "model_owner": { - "title": "Model Owner", - "type": "string" - }, - "save_dir": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Save Dir" - }, - "training_run_id": { - "title": "Training Run Id", - "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" - } - }, - "required": [ - "training_run_id", - "base_model", - "model_owner" - ], - "title": "TrainingRun", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/training_runs/{run_id}/checkpoints": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "get_run_checkpoints", - "path": [ - { - "name": "run_id", - "required": true, - "schema": { - "type": "string" - } - } - ], - "query": [], - "response": { - "$defs": { - "Checkpoint": { - "description": "Twinkle checkpoint model.", - "properties": { - "base_model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Base Model" - }, - "checkpoint_id": { - "title": "Checkpoint Id", - "type": "string" - }, - "checkpoint_type": { - "title": "Checkpoint Type", - "type": "string" - }, - "is_lora": { - "default": false, - "title": "Is Lora", - "type": "boolean" - }, - "lora_rank": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Lora Rank" - }, - "public": { - "default": false, - "title": "Public", - "type": "boolean" - }, - "size_bytes": { - "title": "Size Bytes", - "type": "integer" - }, - "time": { - "format": "date-time", - "title": "Time", - "type": "string" - }, - "train_attn": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Train Attn" - }, - "train_mlp": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Train Mlp" - }, - "train_unembed": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Train Unembed" - }, - "twinkle_path": { - "title": "Twinkle Path", - "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" - } - }, - "required": [ - "checkpoint_id", - "checkpoint_type", - "time", - "size_bytes", - "twinkle_path" - ], - "title": "Checkpoint", - "type": "object" - }, - "Cursor": { - "properties": { - "limit": { - "title": "Limit", - "type": "integer" - }, - "offset": { - "title": "Offset", - "type": "integer" - }, - "total_count": { - "title": "Total Count", - "type": "integer" - } - }, - "required": [ - "limit", - "offset", - "total_count" - ], - "title": "Cursor", - "type": "object" - } - }, - "properties": { - "checkpoints": { - "items": { - "$ref": "#/$defs/Checkpoint" - }, - "title": "Checkpoints", - "type": "array" - }, - "cursor": { - "anyOf": [ - { - "$ref": "#/$defs/Cursor" - }, - { - "type": "null" - } - ], - "default": null - } - }, - "required": [ - "checkpoints" - ], - "title": "CheckpointsListResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/training_runs/{run_id}/checkpoints/{checkpoint_id}": { - "DELETE": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "delete_run_checkpoint", - "path": [ - { - "name": "run_id", - "required": true, - "schema": { - "type": "string" - } - }, - { - "name": "checkpoint_id", - "required": true, - "schema": { - "type": "string" - } - } - ], - "query": [], - "response": { - "properties": { - "message": { - "title": "Message", - "type": "string" - }, - "success": { - "title": "Success", - "type": "boolean" - } - }, - "required": [ - "success", - "message" - ], - "title": "DeleteCheckpointResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/weights_info": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "properties": { - "twinkle_path": { - "title": "Twinkle Path", - "type": "string" - } - }, - "required": [ - "twinkle_path" - ], - "title": "WeightsInfoRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "weights_info", - "path": [], - "query": [], - "response": { - "description": "Twinkle weights info response.", - "properties": { - "base_model": { - "title": "Base Model", - "type": "string" - }, - "is_lora": { - "default": false, - "title": "Is Lora", - "type": "boolean" - }, - "lora_rank": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Lora Rank" - }, - "model_owner": { - "title": "Model Owner", - "type": "string" - }, - "training_run_id": { - "title": "Training Run Id", - "type": "string" - } - }, - "required": [ - "training_run_id", - "base_model", - "model_owner" - ], - "title": "WeightsInfoResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/unload_model": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "model_id": { - "title": "Model Id", - "type": "string" - }, - "type": { - "const": "unload_model", - "default": "unload_model", - "title": "Type", - "type": "string" - } - }, - "required": [ - "model_id" - ], - "title": "UnloadModelRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "unload_model", - "path": [], - "query": [], - "response": {}, - "responses": {}, - "statusCode": 200 - } - }, - "/weights_info": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": {}, - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "weights_info", - "path": [], - "query": [], - "response": { - "description": "Minimal information for loading public checkpoints.", - "properties": { - "base_model": { - "title": "Base Model", - "type": "string" - }, - "is_lora": { - "title": "Is Lora", - "type": "boolean" - }, - "lora_rank": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Lora Rank" - }, - "train_attn": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Train Attn" - }, - "train_mlp": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Train Mlp" - }, - "train_unembed": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Train Unembed" - } - }, - "required": [ - "base_model", - "is_lora" - ], - "title": "WeightsInfoResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - } - } - }, - "model": { - "paths": { - "/healthz": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "model_healthz", - "path": [], - "query": [], - "response": { - "pythonType": "dict" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/tinker/create_model": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "LoraConfig": { - "additionalProperties": false, - "properties": { - "rank": { - "title": "Rank", - "type": "integer" - }, - "seed": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seed" - }, - "train_attn": { - "default": true, - "title": "Train Attn", - "type": "boolean" - }, - "train_mlp": { - "default": true, - "title": "Train Mlp", - "type": "boolean" - }, - "train_unembed": { - "default": true, - "title": "Train Unembed", - "type": "boolean" - } - }, - "required": [ - "rank" - ], - "title": "LoraConfig", - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "base_model": { - "title": "Base Model", - "type": "string" - }, - "lora_config": { - "anyOf": [ - { - "$ref": "#/$defs/LoraConfig" - }, - { - "type": "null" - } - ], - "default": null - }, - "model_seq_id": { - "title": "Model Seq Id", - "type": "integer" - }, - "session_id": { - "title": "Session Id", - "type": "string" - }, - "type": { - "const": "create_model", - "default": "create_model", - "title": "Type", - "type": "string" - }, - "user_metadata": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "User Metadata" - } - }, - "required": [ - "session_id", - "model_seq_id", - "base_model" - ], - "title": "CreateModelRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "create_model", - "path": [], - "query": [], - "response": { - "properties": { - "model_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Id" - }, - "request_id": { - "title": "Request Id", - "type": "string" - } - }, - "required": [ - "request_id" - ], - "title": "UntypedAPIFuture", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/tinker/forward": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "Datum": { - "additionalProperties": false, - "properties": { - "loss_fn_inputs": { - "additionalProperties": { - "$ref": "#/$defs/TensorData" - }, - "title": "Loss Fn Inputs", - "type": "object" - }, - "model_input": { - "$ref": "#/$defs/ModelInput" - } - }, - "required": [ - "loss_fn_inputs", - "model_input" - ], - "title": "Datum", - "type": "object" - }, - "EncodedTextChunk": { - "additionalProperties": false, - "properties": { - "tokens": { - "items": { - "type": "integer" - }, - "title": "Tokens", - "type": "array" - }, - "type": { - "const": "encoded_text", - "default": "encoded_text", - "title": "Type", - "type": "string" - } - }, - "required": [ - "tokens" - ], - "title": "EncodedTextChunk", - "type": "object" - }, - "ForwardBackwardInput": { - "additionalProperties": false, - "properties": { - "data": { - "items": { - "$ref": "#/$defs/Datum" - }, - "title": "Data", - "type": "array" - }, - "loss_fn": { - "enum": [ - "cross_entropy", - "importance_sampling", - "ppo", - "cispo", - "dro" - ], - "title": "Loss Fn", - "type": "string" - }, - "loss_fn_config": { - "anyOf": [ - { - "additionalProperties": { - "type": "number" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Loss Fn Config" - } - }, - "required": [ - "data", - "loss_fn" - ], - "title": "ForwardBackwardInput", - "type": "object" - }, - "ImageAssetPointerChunk": { - "additionalProperties": false, - "properties": { - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "type": { - "const": "image_asset_pointer", - "default": "image_asset_pointer", - "title": "Type", - "type": "string" - } - }, - "required": [ - "format", - "location" - ], - "title": "ImageAssetPointerChunk", - "type": "object" - }, - "ImageChunk": { - "additionalProperties": false, - "properties": { - "data": { - "format": "binary", - "title": "Data", - "type": "string" - }, - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "type": { - "const": "image", - "default": "image", - "title": "Type", - "type": "string" - } - }, - "required": [ - "data", - "format" - ], - "title": "ImageChunk", - "type": "object" - }, - "ModelInput": { - "additionalProperties": false, - "properties": { - "chunks": { - "items": { - "anyOf": [ - { - "$ref": "#/$defs/EncodedTextChunk" - }, - { - "$ref": "#/$defs/ImageAssetPointerChunk" - }, - { - "$ref": "#/$defs/ImageChunk" - } - ] - }, - "title": "Chunks", - "type": "array" - } - }, - "required": [ - "chunks" - ], - "title": "ModelInput", - "type": "object" - }, - "TensorData": { - "additionalProperties": false, - "properties": { - "data": { - "anyOf": [ - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "items": { - "type": "number" - }, - "type": "array" - } - ], - "title": "Data" - }, - "dtype": { - "enum": [ - "int64", - "float32" - ], - "title": "Dtype", - "type": "string" - }, - "shape": { - "anyOf": [ - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Shape" - } - }, - "required": [ - "data", - "dtype" - ], - "title": "TensorData", - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "forward_input": { - "$ref": "#/$defs/ForwardBackwardInput" - }, - "model_id": { - "title": "Model Id", - "type": "string" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - } - }, - "required": [ - "forward_input", - "model_id" - ], - "title": "ForwardRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "forward", - "path": [], - "query": [], - "response": { - "properties": { - "model_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Id" - }, - "request_id": { - "title": "Request Id", - "type": "string" - } - }, - "required": [ - "request_id" - ], - "title": "UntypedAPIFuture", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/tinker/forward_backward": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "Datum": { - "additionalProperties": false, - "properties": { - "loss_fn_inputs": { - "additionalProperties": { - "$ref": "#/$defs/TensorData" - }, - "title": "Loss Fn Inputs", - "type": "object" - }, - "model_input": { - "$ref": "#/$defs/ModelInput" - } - }, - "required": [ - "loss_fn_inputs", - "model_input" - ], - "title": "Datum", - "type": "object" - }, - "EncodedTextChunk": { - "additionalProperties": false, - "properties": { - "tokens": { - "items": { - "type": "integer" - }, - "title": "Tokens", - "type": "array" - }, - "type": { - "const": "encoded_text", - "default": "encoded_text", - "title": "Type", - "type": "string" - } - }, - "required": [ - "tokens" - ], - "title": "EncodedTextChunk", - "type": "object" - }, - "ForwardBackwardInput": { - "additionalProperties": false, - "properties": { - "data": { - "items": { - "$ref": "#/$defs/Datum" - }, - "title": "Data", - "type": "array" - }, - "loss_fn": { - "enum": [ - "cross_entropy", - "importance_sampling", - "ppo", - "cispo", - "dro" - ], - "title": "Loss Fn", - "type": "string" - }, - "loss_fn_config": { - "anyOf": [ - { - "additionalProperties": { - "type": "number" - }, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Loss Fn Config" - } - }, - "required": [ - "data", - "loss_fn" - ], - "title": "ForwardBackwardInput", - "type": "object" - }, - "ImageAssetPointerChunk": { - "additionalProperties": false, - "properties": { - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "type": { - "const": "image_asset_pointer", - "default": "image_asset_pointer", - "title": "Type", - "type": "string" - } - }, - "required": [ - "format", - "location" - ], - "title": "ImageAssetPointerChunk", - "type": "object" - }, - "ImageChunk": { - "additionalProperties": false, - "properties": { - "data": { - "format": "binary", - "title": "Data", - "type": "string" - }, - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "type": { - "const": "image", - "default": "image", - "title": "Type", - "type": "string" - } - }, - "required": [ - "data", - "format" - ], - "title": "ImageChunk", - "type": "object" - }, - "ModelInput": { - "additionalProperties": false, - "properties": { - "chunks": { - "items": { - "anyOf": [ - { - "$ref": "#/$defs/EncodedTextChunk" - }, - { - "$ref": "#/$defs/ImageAssetPointerChunk" - }, - { - "$ref": "#/$defs/ImageChunk" - } - ] - }, - "title": "Chunks", - "type": "array" - } - }, - "required": [ - "chunks" - ], - "title": "ModelInput", - "type": "object" - }, - "TensorData": { - "additionalProperties": false, - "properties": { - "data": { - "anyOf": [ - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "items": { - "type": "number" - }, - "type": "array" - } - ], - "title": "Data" - }, - "dtype": { - "enum": [ - "int64", - "float32" - ], - "title": "Dtype", - "type": "string" - }, - "shape": { - "anyOf": [ - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Shape" - } - }, - "required": [ - "data", - "dtype" - ], - "title": "TensorData", - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "forward_backward_input": { - "$ref": "#/$defs/ForwardBackwardInput" - }, - "model_id": { - "title": "Model Id", - "type": "string" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - } - }, - "required": [ - "forward_backward_input", - "model_id" - ], - "title": "ForwardBackwardRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "forward_backward", - "path": [], - "query": [], - "response": { - "properties": { - "model_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Id" - }, - "request_id": { - "title": "Request Id", - "type": "string" - } - }, - "required": [ - "request_id" - ], - "title": "UntypedAPIFuture", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/tinker/get_info": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "model_id": { - "title": "Model Id", - "type": "string" - }, - "type": { - "const": "get_info", - "default": "get_info", - "title": "Type", - "type": "string" - } - }, - "required": [ - "model_id" - ], - "title": "GetInfoRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "get_info", - "path": [], - "query": [], - "response": { - "$defs": { - "ModelData": { - "description": "Metadata about a model's architecture and configuration.", - "properties": { - "arch": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Arch" - }, - "model_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Name" - }, - "tokenizer_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Tokenizer Id" - } - }, - "title": "ModelData", - "type": "object" - } - }, - "description": "Response containing information about a training client's model.", - "properties": { - "is_lora": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Is Lora" - }, - "lora_rank": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Lora Rank" - }, - "model_data": { - "$ref": "#/$defs/ModelData" - }, - "model_id": { - "title": "Model Id", - "type": "string" - }, - "model_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Name" - }, - "type": { - "anyOf": [ - { - "const": "get_info", - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Type" - } - }, - "required": [ - "model_data", - "model_id" - ], - "title": "GetInfoResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/tinker/load_weights": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "model_id": { - "title": "Model Id", - "type": "string" - }, - "optimizer": { - "title": "Optimizer", - "type": "boolean" - }, - "path": { - "title": "Path", - "type": "string" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - }, - "type": { - "const": "load_weights", - "default": "load_weights", - "title": "Type", - "type": "string" - } - }, - "required": [ - "model_id", - "path", - "optimizer" - ], - "title": "LoadWeightsRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "load_weights", - "path": [], - "query": [], - "response": { - "properties": { - "model_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Id" - }, - "request_id": { - "title": "Request Id", - "type": "string" - } - }, - "required": [ - "request_id" - ], - "title": "UntypedAPIFuture", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/tinker/optim_step": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "AdamParams": { - "additionalProperties": false, - "properties": { - "beta1": { - "default": 0.9, - "title": "Beta1", - "type": "number" - }, - "beta2": { - "default": 0.95, - "title": "Beta2", - "type": "number" - }, - "eps": { - "default": 1e-12, - "title": "Eps", - "type": "number" - }, - "grad_clip_norm": { - "default": 0.0, - "title": "Grad Clip Norm", - "type": "number" - }, - "learning_rate": { - "default": 0.0001, - "title": "Learning Rate", - "type": "number" - }, - "weight_decay": { - "default": 0.0, - "title": "Weight Decay", - "type": "number" - } - }, - "title": "AdamParams", - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "adam_params": { - "$ref": "#/$defs/AdamParams" - }, - "model_id": { - "title": "Model Id", - "type": "string" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - }, - "type": { - "const": "optim_step", - "default": "optim_step", - "title": "Type", - "type": "string" - } - }, - "required": [ - "adam_params", - "model_id" - ], - "title": "OptimStepRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "optim_step", - "path": [], - "query": [], - "response": { - "properties": { - "model_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Id" - }, - "request_id": { - "title": "Request Id", - "type": "string" - } - }, - "required": [ - "request_id" - ], - "title": "UntypedAPIFuture", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/tinker/save_weights": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "model_id": { - "title": "Model Id", - "type": "string" - }, - "path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Path" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - }, - "ttl_seconds": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Ttl Seconds" - }, - "type": { - "const": "save_weights", - "default": "save_weights", - "title": "Type", - "type": "string" - } - }, - "required": [ - "model_id" - ], - "title": "SaveWeightsRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "save_weights", - "path": [], - "query": [], - "response": { - "properties": { - "model_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Id" - }, - "request_id": { - "title": "Request Id", - "type": "string" - } - }, - "required": [ - "request_id" - ], - "title": "UntypedAPIFuture", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/tinker/save_weights_for_sampler": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "model_id": { - "title": "Model Id", - "type": "string" - }, - "path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Path" - }, - "sampling_session_seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sampling Session Seq Id" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - }, - "ttl_seconds": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Ttl Seconds" - }, - "type": { - "const": "save_weights_for_sampler", - "default": "save_weights_for_sampler", - "title": "Type", - "type": "string" - } - }, - "required": [ - "model_id" - ], - "title": "SaveWeightsForSamplerRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "save_weights_for_sampler", - "path": [], - "query": [], - "response": { - "properties": { - "model_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Id" - }, - "request_id": { - "title": "Request Id", - "type": "string" - } - }, - "required": [ - "request_id" - ], - "title": "UntypedAPIFuture", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/tinker/unload_model": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": false, - "properties": { - "model_id": { - "title": "Model Id", - "type": "string" - }, - "type": { - "const": "unload_model", - "default": "unload_model", - "title": "Type", - "type": "string" - } - }, - "required": [ - "model_id" - ], - "title": "UnloadModelRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "unload_model", - "path": [], - "query": [], - "response": { - "properties": { - "model_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Id" - }, - "request_id": { - "title": "Request Id", - "type": "string" - } - }, - "required": [ - "request_id" - ], - "title": "UntypedAPIFuture", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/add_adapter_to_model": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "config": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Config" - }, - "save_dir": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Save Dir" - } - }, - "required": [ - "adapter_name" - ], - "title": "AddAdapterRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "add_adapter_to_model", - "path": [], - "query": [], - "response": { - "description": "Response body for the /add_adapter_to_sampler endpoint.", - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "status": { - "default": "ok", - "title": "Status", - "type": "string" - } - }, - "required": [ - "adapter_name" - ], - "title": "AddAdapterResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/add_metric": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "is_training": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Is Training" - }, - "metric_cls": { - "title": "Metric Cls", - "type": "string" - } - }, - "required": [ - "metric_cls", - "adapter_name" - ], - "title": "AddMetricRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "add_metric", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/apply_patch": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "patch_cls": { - "title": "Patch Cls", - "type": "string" - } - }, - "required": [ - "patch_cls", - "adapter_name" - ], - "title": "ApplyPatchRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "apply_patch", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/backward": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - } - }, - "required": [ - "adapter_name" - ], - "title": "AdapterRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "backward", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/calculate_loss": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - } - }, - "required": [ - "adapter_name" - ], - "title": "AdapterRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "calculate_loss", - "path": [], - "query": [], - "response": { - "description": "Response for /calculate_loss endpoint (returns float).", - "properties": { - "result": { - "title": "Result", - "type": "number" - } - }, - "required": [ - "result" - ], - "title": "CalculateLossResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/calculate_metric": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "is_training": { - "default": true, - "title": "Is Training", - "type": "boolean" - } - }, - "required": [ - "adapter_name" - ], - "title": "CalculateMetricRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "calculate_metric", - "path": [], - "query": [], - "response": { - "description": "Response for /calculate_metric endpoint (returns Dict).", - "properties": { - "result": { - "additionalProperties": true, - "title": "Result", - "type": "object" - } - }, - "required": [ - "result" - ], - "title": "CalculateMetricResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/clip_grad_and_step": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "max_grad_norm": { - "default": 1.0, - "title": "Max Grad Norm", - "type": "number" - }, - "norm_type": { - "default": 2, - "title": "Norm Type", - "type": "integer" - } - }, - "required": [ - "adapter_name" - ], - "title": "ClipGradAndStepRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "clip_grad_and_step", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/clip_grad_norm": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - } - }, - "required": [ - "adapter_name" - ], - "title": "AdapterRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "clip_grad_norm", - "path": [], - "query": [], - "response": { - "description": "Response for /clip_grad_norm endpoint (returns float as str).", - "properties": { - "result": { - "title": "Result", - "type": "string" - } - }, - "required": [ - "result" - ], - "title": "ClipGradNormResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/create": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": {}, - "title": "CreateRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "create", - "path": [], - "query": [], - "response": { - "description": "Response for /create endpoint.", - "properties": { - "status": { - "default": "ok", - "title": "Status", - "type": "string" - } - }, - "title": "CreateResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/forward": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "inputs": { - "title": "Inputs" - } - }, - "required": [ - "inputs", - "adapter_name" - ], - "title": "ForwardRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "forward", - "path": [], - "query": [], - "response": { - "description": "Response for /forward and /forward_only endpoints (returns ModelOutput).", - "properties": { - "result": { - "title": "Result" - } - }, - "required": [ - "result" - ], - "title": "ForwardResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/forward_backward": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "inputs": { - "title": "Inputs" - } - }, - "required": [ - "inputs", - "adapter_name" - ], - "title": "ForwardRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "forward_backward", - "path": [], - "query": [], - "response": { - "description": "Response for /forward_backward endpoint (returns ModelOutput).", - "properties": { - "result": { - "title": "Result" - } - }, - "required": [ - "result" - ], - "title": "ForwardBackwardResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/forward_backward_from_data_plane": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "DataRef": { - "description": "Opaque reference to rows stored in the server-side TransferQueue.", - "properties": { - "fields": { - "items": { - "type": "string" - }, - "title": "Fields", - "type": "array" - }, - "kind": { - "default": "data", - "title": "Kind", - "type": "string" - }, - "num_tokens": { - "default": 0, - "title": "Num Tokens", - "type": "integer" - }, - "ref_id": { - "title": "Ref Id", - "type": "string" - }, - "size": { - "title": "Size", - "type": "integer" - } - }, - "required": [ - "ref_id", - "size" - ], - "title": "DataRef", - "type": "object" - } - }, - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "input_field": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Input Field" - }, - "input_refs": { - "items": { - "$ref": "#/$defs/DataRef" - }, - "minItems": 1, - "title": "Input Refs", - "type": "array" - }, - "kwarg_fields": { - "additionalProperties": { - "type": "string" - }, - "title": "Kwarg Fields", - "type": "object" - } - }, - "required": [ - "input_refs", - "adapter_name" - ], - "title": "DataPlaneForwardRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "forward_backward_from_data_plane", - "path": [], - "query": [], - "response": { - "description": "Response for /forward_backward endpoint (returns ModelOutput).", - "properties": { - "result": { - "title": "Result" - } - }, - "required": [ - "result" - ], - "title": "ForwardBackwardResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/forward_from_data_plane": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "DataRef": { - "description": "Opaque reference to rows stored in the server-side TransferQueue.", - "properties": { - "fields": { - "items": { - "type": "string" - }, - "title": "Fields", - "type": "array" - }, - "kind": { - "default": "data", - "title": "Kind", - "type": "string" - }, - "num_tokens": { - "default": 0, - "title": "Num Tokens", - "type": "integer" - }, - "ref_id": { - "title": "Ref Id", - "type": "string" - }, - "size": { - "title": "Size", - "type": "integer" - } - }, - "required": [ - "ref_id", - "size" - ], - "title": "DataRef", - "type": "object" - } - }, - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "input_field": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Input Field" - }, - "input_refs": { - "items": { - "$ref": "#/$defs/DataRef" - }, - "minItems": 1, - "title": "Input Refs", - "type": "array" - }, - "kwarg_fields": { - "additionalProperties": { - "type": "string" - }, - "title": "Kwarg Fields", - "type": "object" - } - }, - "required": [ - "input_refs", - "adapter_name" - ], - "title": "DataPlaneForwardRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "forward_from_data_plane", - "path": [], - "query": [], - "response": { - "description": "Response for /forward and /forward_only endpoints (returns ModelOutput).", - "properties": { - "result": { - "title": "Result" - } - }, - "required": [ - "result" - ], - "title": "ForwardResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/forward_only": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Adapter Name" - }, - "inputs": { - "title": "Inputs" - } - }, - "required": [ - "inputs" - ], - "title": "ForwardOnlyRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "forward_only", - "path": [], - "query": [], - "response": { - "description": "Response for /forward and /forward_only endpoints (returns ModelOutput).", - "properties": { - "result": { - "title": "Result" - } - }, - "required": [ - "result" - ], - "title": "ForwardResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/forward_only_from_data_plane": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "DataRef": { - "description": "Opaque reference to rows stored in the server-side TransferQueue.", - "properties": { - "fields": { - "items": { - "type": "string" - }, - "title": "Fields", - "type": "array" - }, - "kind": { - "default": "data", - "title": "Kind", - "type": "string" - }, - "num_tokens": { - "default": 0, - "title": "Num Tokens", - "type": "integer" - }, - "ref_id": { - "title": "Ref Id", - "type": "string" - }, - "size": { - "title": "Size", - "type": "integer" - } - }, - "required": [ - "ref_id", - "size" - ], - "title": "DataRef", - "type": "object" - } - }, - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "input_field": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Input Field" - }, - "input_refs": { - "items": { - "$ref": "#/$defs/DataRef" - }, - "minItems": 1, - "title": "Input Refs", - "type": "array" - }, - "kwarg_fields": { - "additionalProperties": { - "type": "string" - }, - "title": "Kwarg Fields", - "type": "object" - }, - "output_fields": { - "additionalProperties": { - "type": "string" - }, - "title": "Output Fields", - "type": "object" - }, - "output_ref": { - "anyOf": [ - { - "$ref": "#/$defs/DataRef" - }, - { - "type": "null" - } - ], - "default": null - } - }, - "required": [ - "input_refs", - "adapter_name" - ], - "title": "DataPlaneForwardOnlyRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "forward_only_from_data_plane", - "path": [], - "query": [], - "response": { - "description": "Response for /forward and /forward_only endpoints (returns ModelOutput).", - "properties": { - "result": { - "title": "Result" - } - }, - "required": [ - "result" - ], - "title": "ForwardResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/get_state_dict": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - } - }, - "required": [ - "adapter_name" - ], - "title": "GetStateDictRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "get_state_dict", - "path": [], - "query": [], - "response": { - "description": "Response for /get_state_dict endpoint (returns Dict).", - "properties": { - "result": { - "additionalProperties": true, - "title": "Result", - "type": "object" - } - }, - "required": [ - "result" - ], - "title": "GetStateDictResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/get_train_configs": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - } - }, - "required": [ - "adapter_name" - ], - "title": "AdapterRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "get_train_configs", - "path": [], - "query": [], - "response": { - "description": "Response for /get_train_configs endpoint (returns str).", - "properties": { - "result": { - "title": "Result", - "type": "string" - } - }, - "required": [ - "result" - ], - "title": "GetTrainConfigsResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/load": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "load_optimizer": { - "default": false, - "title": "Load Optimizer", - "type": "boolean" - }, - "name": { - "title": "Name", - "type": "string" - } - }, - "required": [ - "adapter_name", - "name" - ], - "title": "LoadRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "load", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/lr_step": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - } - }, - "required": [ - "adapter_name" - ], - "title": "AdapterRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "lr_step", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/remove_adapter": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - } - }, - "required": [ - "adapter_name" - ], - "title": "AdapterRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "remove_adapter", - "path": [], - "query": [], - "response": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/resume_from_checkpoint": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "description": "Request for /resume_from_checkpoint endpoint.", - "properties": { - "adapter_name": { - "default": "", - "title": "Adapter Name", - "type": "string" - }, - "name": { - "title": "Name", - "type": "string" - }, - "resume_only_model": { - "default": false, - "title": "Resume Only Model", - "type": "boolean" - } - }, - "required": [ - "name" - ], - "title": "ResumeFromCheckpointRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "resume_from_checkpoint", - "path": [], - "query": [], - "response": { - "description": "Response for /resume_from_checkpoint endpoint.", - "properties": { - "result": { - "additionalProperties": true, - "title": "Result", - "type": "object" - } - }, - "required": [ - "result" - ], - "title": "TrainingProgressResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/save": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "is_sampler": { - "default": false, - "title": "Is Sampler", - "type": "boolean" - }, - "name": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Name" - }, - "save_optimizer": { - "default": false, - "title": "Save Optimizer", - "type": "boolean" - } - }, - "required": [ - "adapter_name" - ], - "title": "SaveRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "save", - "path": [], - "query": [], - "response": { - "description": "Response for /save endpoint (returns twinkle path + checkpoint dir).", - "properties": { - "checkpoint_dir": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Checkpoint Dir" - }, - "twinkle_path": { - "title": "Twinkle Path", - "type": "string" - } - }, - "required": [ - "twinkle_path" - ], - "title": "SaveResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/set_loss": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "loss_cls": { - "title": "Loss Cls", - "type": "string" - } - }, - "required": [ - "loss_cls", - "adapter_name" - ], - "title": "SetLossRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "set_loss", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/set_lr_scheduler": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "scheduler_cls": { - "title": "Scheduler Cls", - "type": "string" - } - }, - "required": [ - "scheduler_cls", - "adapter_name" - ], - "title": "SetLrSchedulerRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "set_lr_scheduler", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/set_optimizer": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "optimizer_cls": { - "title": "Optimizer Cls", - "type": "string" - } - }, - "required": [ - "optimizer_cls", - "adapter_name" - ], - "title": "SetOptimizerRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "set_optimizer", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/set_processor": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "processor_cls": { - "title": "Processor Cls", - "type": "string" - } - }, - "required": [ - "processor_cls", - "adapter_name" - ], - "title": "SetProcessorRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "set_processor", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/set_template": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "template_cls": { - "title": "Template Cls", - "type": "string" - } - }, - "required": [ - "template_cls", - "adapter_name" - ], - "title": "SetTemplateRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "set_template", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/step": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - } - }, - "required": [ - "adapter_name" - ], - "title": "AdapterRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "step", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/upload_status/{request_id}": { - "GET": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "upload_status", - "path": [ - { - "name": "request_id", - "required": true, - "schema": { - "type": "string" - } - } - ], - "query": [], - "response": { - "description": "Response for /upload_status/{request_id} endpoint.", - "properties": { - "error": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Error" - }, - "request_id": { - "title": "Request Id", - "type": "string" - }, - "status": { - "title": "Status", - "type": "string" - } - }, - "required": [ - "request_id", - "status" - ], - "title": "UploadStatusResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/upload_to_hub": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "async_upload": { - "default": false, - "title": "Async Upload", - "type": "boolean" - }, - "checkpoint_dir": { - "anyOf": [ - { - "type": "string" - }, - { - "additionalProperties": true, - "type": "object" - } - ], - "title": "Checkpoint Dir" - }, - "hub_model_id": { - "title": "Hub Model Id", - "type": "string" - }, - "hub_token": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Hub Token" - } - }, - "required": [ - "checkpoint_dir", - "hub_model_id" - ], - "title": "UploadToHubRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "upload_to_hub", - "path": [], - "query": [], - "response": { - "description": "Response for /upload_to_hub endpoint.", - "properties": { - "request_id": { - "title": "Request Id", - "type": "string" - } - }, - "required": [ - "request_id" - ], - "title": "UploadToHubResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/zero_grad": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - } - }, - "required": [ - "adapter_name" - ], - "title": "AdapterRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "zero_grad", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - } - } - }, - "processor": { - "paths": { - "/twinkle/call": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "function": { - "title": "Function", - "type": "string" - }, - "processor_id": { - "title": "Processor Id", - "type": "string" - } - }, - "required": [ - "processor_id", - "function" - ], - "title": "ProcessorCallRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "call", - "path": [], - "query": [], - "response": { - "description": "Response body for the /call endpoint.", - "properties": { - "result": { - "title": "Result" - } - }, - "required": [ - "result" - ], - "title": "ProcessorCallResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/create": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "class_type": { - "title": "Class Type", - "type": "string" - }, - "processor_type": { - "title": "Processor Type", - "type": "string" - } - }, - "required": [ - "processor_type", - "class_type" - ], - "title": "ProcessorCreateRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "create", - "path": [], - "query": [], - "response": { - "description": "Response body for the /create endpoint.", - "properties": { - "processor_id": { - "title": "Processor Id", - "type": "string" - } - }, - "required": [ - "processor_id" - ], - "title": "ProcessorCreateResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - } - } - }, - "sampler": { - "paths": { - "/tinker/asample": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "EncodedTextChunk": { - "additionalProperties": false, - "properties": { - "tokens": { - "items": { - "type": "integer" - }, - "title": "Tokens", - "type": "array" - }, - "type": { - "const": "encoded_text", - "default": "encoded_text", - "title": "Type", - "type": "string" - } - }, - "required": [ - "tokens" - ], - "title": "EncodedTextChunk", - "type": "object" - }, - "ImageAssetPointerChunk": { - "additionalProperties": false, - "properties": { - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "location": { - "title": "Location", - "type": "string" - }, - "type": { - "const": "image_asset_pointer", - "default": "image_asset_pointer", - "title": "Type", - "type": "string" - } - }, - "required": [ - "format", - "location" - ], - "title": "ImageAssetPointerChunk", - "type": "object" - }, - "ImageChunk": { - "additionalProperties": false, - "properties": { - "data": { - "format": "binary", - "title": "Data", - "type": "string" - }, - "expected_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Expected Tokens" - }, - "format": { - "enum": [ - "png", - "jpeg" - ], - "title": "Format", - "type": "string" - }, - "type": { - "const": "image", - "default": "image", - "title": "Type", - "type": "string" - } - }, - "required": [ - "data", - "format" - ], - "title": "ImageChunk", - "type": "object" - }, - "ModelInput": { - "additionalProperties": false, - "properties": { - "chunks": { - "items": { - "anyOf": [ - { - "$ref": "#/$defs/EncodedTextChunk" - }, - { - "$ref": "#/$defs/ImageAssetPointerChunk" - }, - { - "$ref": "#/$defs/ImageChunk" - } - ] - }, - "title": "Chunks", - "type": "array" - } - }, - "required": [ - "chunks" - ], - "title": "ModelInput", - "type": "object" - }, - "SamplingParams": { - "properties": { - "max_tokens": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Max Tokens" - }, - "seed": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seed" - }, - "stop": { - "anyOf": [ - { - "type": "string" - }, - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Stop" - }, - "temperature": { - "default": 1, - "title": "Temperature", - "type": "number" - }, - "top_k": { - "default": -1, - "title": "Top K", - "type": "integer" - }, - "top_p": { - "default": 1, - "title": "Top P", - "type": "number" - } - }, - "title": "SamplingParams", - "type": "object" - } - }, - "additionalProperties": false, - "properties": { - "base_model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Base Model" - }, - "model_path": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Path" - }, - "num_samples": { - "default": 1, - "title": "Num Samples", - "type": "integer" - }, - "prompt": { - "$ref": "#/$defs/ModelInput" - }, - "prompt_logprobs": { - "anyOf": [ - { - "type": "boolean" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Prompt Logprobs" - }, - "sampling_params": { - "$ref": "#/$defs/SamplingParams" - }, - "sampling_session_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sampling Session Id" - }, - "seq_id": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Seq Id" - }, - "topk_prompt_logprobs": { - "default": 0, - "title": "Topk Prompt Logprobs", - "type": "integer" - }, - "type": { - "const": "sample", - "default": "sample", - "title": "Type", - "type": "string" - } - }, - "required": [ - "prompt", - "sampling_params" - ], - "title": "SampleRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "asample", - "path": [], - "query": [], - "response": { - "properties": { - "model_id": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Model Id" - }, - "request_id": { - "title": "Request Id", - "type": "string" - } - }, - "required": [ - "request_id" - ], - "title": "UntypedAPIFuture", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/add_adapter_to_sampler": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "config": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Config" - }, - "save_dir": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Save Dir" - } - }, - "required": [ - "adapter_name" - ], - "title": "AddAdapterRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "add_adapter_to_sampler", - "path": [], - "query": [], - "response": { - "description": "Response body for the /add_adapter_to_sampler endpoint.", - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "status": { - "default": "ok", - "title": "Status", - "type": "string" - } - }, - "required": [ - "adapter_name" - ], - "title": "AddAdapterResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/apply_patch": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "patch_cls": { - "title": "Patch Cls", - "type": "string" - } - }, - "required": [ - "patch_cls", - "adapter_name" - ], - "title": "ApplyPatchRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "apply_patch", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/create": { - "POST": { - "body": [], - "cookies": [], - "headers": [], - "operationId": "create", - "path": [], - "query": [], - "response": { - "description": "Response for /create endpoint.", - "properties": { - "status": { - "default": "ok", - "title": "Status", - "type": "string" - } - }, - "title": "CreateResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/sample": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "description": "Request body for the /sample endpoint.", - "properties": { - "adapter_name": { - "default": "", - "description": "Adapter name for LoRA inference", - "title": "Adapter Name", - "type": "string" - }, - "adapter_uri": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Adapter URI (twinkle:// path or local path) for LoRA inference", - "title": "Adapter Uri" - }, - "inputs": { - "description": "List of Trajectory or InputFeature dicts", - "title": "Inputs" - }, - "sampling_params": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Sampling parameters (max_tokens, temperature, num_samples, etc.)", - "title": "Sampling Params" - } - }, - "required": [ - "inputs" - ], - "title": "SampleRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "sample", - "path": [], - "query": [], - "response": { - "$defs": { - "SampleResponseModel": { - "description": "Mirroring twinkle.data_format.SampleResponse.", - "properties": { - "prompt_logprobs": { - "anyOf": [ - { - "items": { - "anyOf": [ - { - "type": "number" - }, - { - "type": "null" - } - ] - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Prompt Logprobs" - }, - "prompt_token_ids": { - "anyOf": [ - { - "items": { - "type": "integer" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Token IDs of the prompt the sequences continue", - "title": "Prompt Token Ids" - }, - "sequences": { - "description": "List of sampled sequences", - "items": { - "$ref": "#/$defs/SampledSequenceModel" - }, - "title": "Sequences", - "type": "array" - }, - "topk_prompt_logprobs": { - "anyOf": [ - { - "items": { - "anyOf": [ - { - "items": { - "maxItems": 2, - "minItems": 2, - "prefixItems": [ - { - "type": "integer" - }, - { - "type": "number" - } - ], - "type": "array" - }, - "type": "array" - }, - { - "type": "null" - } - ] - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Topk Prompt Logprobs" - } - }, - "required": [ - "sequences" - ], - "title": "SampleResponseModel", - "type": "object" - }, - "SampledSequenceModel": { - "description": "A single sampled sequence, mirroring twinkle.data_format.SampledSequence.", - "properties": { - "decoded": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Decoded text of the sampled sequence", - "title": "Decoded" - }, - "logprobs": { - "anyOf": [ - { - "items": { - "anyOf": [ - { - "items": { - "maxItems": 2, - "minItems": 2, - "prefixItems": [ - { - "type": "integer" - }, - { - "type": "number" - } - ], - "type": "array" - }, - "type": "array" - }, - { - "type": "null" - } - ] - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Per-token log-probabilities", - "title": "Logprobs" - }, - "new_input_feature": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Updated InputFeature after sampling (input_ids, labels, etc.)", - "title": "New Input Feature" - }, - "stop_reason": { - "description": "Stop reason: 'length' or 'stop'", - "enum": [ - "length", - "stop", - "abort", - "error" - ], - "title": "Stop Reason", - "type": "string" - }, - "tokens": { - "description": "Token IDs of the sampled sequence", - "items": { - "type": "integer" - }, - "title": "Tokens", - "type": "array" - } - }, - "required": [ - "stop_reason", - "tokens" - ], - "title": "SampledSequenceModel", - "type": "object" - } - }, - "description": "Response body for the /sample endpoint", - "properties": { - "samples": { - "description": "List of sample responses", - "items": { - "$ref": "#/$defs/SampleResponseModel" - }, - "title": "Samples", - "type": "array" - } - }, - "required": [ - "samples" - ], - "title": "SampleResponseModelList", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/sample_stream": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "description": "Request body for the /sample endpoint.", - "properties": { - "adapter_name": { - "default": "", - "description": "Adapter name for LoRA inference", - "title": "Adapter Name", - "type": "string" - }, - "adapter_uri": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Adapter URI (twinkle:// path or local path) for LoRA inference", - "title": "Adapter Uri" - }, - "inputs": { - "description": "List of Trajectory or InputFeature dicts", - "title": "Inputs" - }, - "sampling_params": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Sampling parameters (max_tokens, temperature, num_samples, etc.)", - "title": "Sampling Params" - } - }, - "required": [ - "inputs" - ], - "title": "SampleRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "sample_stream", - "path": [], - "query": [], - "response": { - "type": "null" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/sample_to_data_plane": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "$defs": { - "DataRef": { - "description": "Opaque reference to rows stored in the server-side TransferQueue.", - "properties": { - "fields": { - "items": { - "type": "string" - }, - "title": "Fields", - "type": "array" - }, - "kind": { - "default": "data", - "title": "Kind", - "type": "string" - }, - "num_tokens": { - "default": 0, - "title": "Num Tokens", - "type": "integer" - }, - "ref_id": { - "title": "Ref Id", - "type": "string" - }, - "size": { - "title": "Size", - "type": "integer" - } - }, - "required": [ - "ref_id", - "size" - ], - "title": "DataRef", - "type": "object" - } - }, - "properties": { - "adapter_name": { - "default": "", - "title": "Adapter Name", - "type": "string" - }, - "adapter_uri": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Adapter Uri" - }, - "group_ids": { - "anyOf": [ - { - "items": { - "type": "string" - }, - "type": "array" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Group Ids" - }, - "input_ref": { - "anyOf": [ - { - "$ref": "#/$defs/DataRef" - }, - { - "type": "null" - } - ], - "default": null - }, - "inputs": { - "default": null, - "title": "Inputs" - }, - "num_samples": { - "default": 1, - "title": "Num Samples", - "type": "integer" - }, - "policy_version": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Policy Version" - }, - "sampling_params": { - "anyOf": [ - { - "additionalProperties": true, - "type": "object" - }, - { - "type": "null" - } - ], - "default": null, - "title": "Sampling Params" - } - }, - "title": "DataPlaneSampleRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "sample_to_data_plane", - "path": [], - "query": [], - "response": { - "description": "Opaque reference to rows stored in the server-side TransferQueue.", - "properties": { - "fields": { - "items": { - "type": "string" - }, - "title": "Fields", - "type": "array" - }, - "kind": { - "default": "data", - "title": "Kind", - "type": "string" - }, - "num_tokens": { - "default": 0, - "title": "Num Tokens", - "type": "integer" - }, - "ref_id": { - "title": "Ref Id", - "type": "string" - }, - "size": { - "title": "Size", - "type": "integer" - } - }, - "required": [ - "ref_id", - "size" - ], - "title": "DataRef", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/set_template": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "additionalProperties": true, - "properties": { - "adapter_name": { - "title": "Adapter Name", - "type": "string" - }, - "template_cls": { - "title": "Template Cls", - "type": "string" - } - }, - "required": [ - "template_cls", - "adapter_name" - ], - "title": "SetTemplateRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "set_template", - "path": [], - "query": [], - "response": { - "description": "Response for /set_template endpoint.", - "properties": { - "status": { - "default": "ok", - "title": "Status", - "type": "string" - } - }, - "title": "SetTemplateResponse", - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - }, - "/twinkle/unload_adapter_paths": { - "POST": { - "body": [ - { - "name": "body", - "required": true, - "schema": { - "properties": { - "adapter_paths": { - "items": { - "type": "string" - }, - "title": "Adapter Paths", - "type": "array" - } - }, - "required": [ - "adapter_paths" - ], - "title": "UnloadAdapterPathsRequest", - "type": "object" - } - } - ], - "cookies": [], - "headers": [], - "operationId": "unload_adapter_paths", - "path": [], - "query": [], - "response": { - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "responses": {}, - "statusCode": 200 - } - } - } - } -} diff --git a/tests/server/contract/client_api_harness.py b/tests/server/contract/client_api_harness.py index 59d4ac5a..975b579c 100644 --- a/tests/server/contract/client_api_harness.py +++ b/tests/server/contract/client_api_harness.py @@ -189,7 +189,7 @@ def _extract_app_surface(app: FastAPI) -> dict[str, Any]: 'statusCode': route.status_code or 200, } for method in sorted(route.methods & _HTTP_METHODS): - client_path = re.sub(r'{([^}:]+):[^}]+}', r'{\1}', route.path) + client_path = _client_path(route.path) paths.setdefault(client_path, {})[method] = operation return {'paths': paths} @@ -203,10 +203,65 @@ def extract_full_surface() -> dict[str, Any]: return surface -# ----- Baseline I/O -------------------------------------------------------- # +def _client_path(route_path: str) -> str: + """Strip FastAPI path-converter suffixes so ``{id:path}`` reads as ``{id}``.""" + return re.sub(r'{([^}:]+):[^}]+}', r'{\1}', route_path) + +def _model_name(annotation: Any) -> str | None: + """A stable, human-readable name for a request/response model annotation.""" + if annotation is None: + return None + return getattr(annotation, '__qualname__', None) or repr(annotation) + + +def extract_route_inventory() -> dict[str, dict[str, Any]]: + """A compact, reviewable projection of the wire surface. + + One entry per route -- ``" " -> {response, body, statusCode}`` -- + naming the model classes instead of inlining their field schemas. + + This is the projection that gets committed. A route appearing, disappearing, or + changing its response/body model shows up as a few readable lines in a PR diff, + whereas the full field-level surface is ~8k lines and nobody reads that diff. + The trade-off is explicit: this catches route-level and model-level changes, not + field-level drift inside a model. + """ + inventory: dict[str, dict[str, Any]] = {} + for name, builder in APP_BUILDERS.items(): + app = builder() + routes: dict[str, Any] = {} + for route in app.routes: + if not isinstance(route, APIRoute): + continue + client_path = _client_path(route.path) + body = [_model_name(field.field_info.annotation) for field in route.dependant.body_params] + for method in sorted(route.methods & _HTTP_METHODS): + routes[f'{method} {client_path}'] = { + 'response': _model_name(route.response_model), + 'body': body, + 'statusCode': route.status_code or 200, + } + inventory[name] = routes + return inventory + + +# ----- Snapshot I/O -------------------------------------------------------- # + +# The full field-level surface (:func:`extract_full_surface`). A GENERATED artifact, +# deliberately NOT committed: an 8k-line diff on every intentional wire change is noise +# nobody reads. Being regenerated from the code under test, it cannot by itself detect an +# unintended change -- ``ROUTES_PATH`` is the guard that can. Keep that asymmetry in mind +# before treating a green baseline test as evidence of anything. BASELINE_PATH = Path(__file__).parent / 'client_api_baseline.json' +# The compact route inventory (:func:`extract_route_inventory`). COMMITTED to git: this +# is the actual regression guard, so it has to stay tracked for the guard to mean +# anything. +ROUTES_PATH = Path(__file__).parent / 'client_api_routes.json' + +_REGEN_HINT = 'Regenerate with: python -m tests.server.contract.update_baseline' + def write_baseline(path: Path | None = None) -> Path: """Snapshot the current client-API surface to ``client_api_baseline.json``.""" @@ -216,6 +271,28 @@ def write_baseline(path: Path | None = None) -> Path: return p +def write_route_inventory(path: Path | None = None) -> Path: + """Snapshot the compact route inventory to ``client_api_routes.json``.""" + p = Path(path) if path is not None else ROUTES_PATH + p.write_text(json.dumps(extract_route_inventory(), indent=2, sort_keys=True) + '\n') + return p + + def load_baseline(path: Path | None = None) -> dict[str, Any]: + """Load the generated full surface, failing with a fix hint rather than a bare OSError.""" p = Path(path) if path is not None else BASELINE_PATH + if not p.is_file(): + raise FileNotFoundError(f'Contract baseline missing: {p}\n' + f'It is a generated artifact and is deliberately not committed. ' + f'{_REGEN_HINT}') + return json.loads(p.read_text()) + + +def load_route_inventory(path: Path | None = None) -> dict[str, Any]: + """Load the committed route inventory, failing loudly if it went missing.""" + p = Path(path) if path is not None else ROUTES_PATH + if not p.is_file(): + raise FileNotFoundError(f'Committed route inventory missing: {p}\n' + f'This file IS tracked by git -- restore it instead of regenerating ' + f'blindly, or the guard silently becomes a tautology. {_REGEN_HINT}') return json.loads(p.read_text()) diff --git a/tests/server/contract/client_api_routes.json b/tests/server/contract/client_api_routes.json new file mode 100644 index 00000000..8af1e5c7 --- /dev/null +++ b/tests/server/contract/client_api_routes.json @@ -0,0 +1,628 @@ +{ + "data_plane": { + "POST /twinkle/append": { + "body": [ + "DataAppendRequest" + ], + "response": "DataRef", + "statusCode": 200 + }, + "POST /twinkle/get": { + "body": [ + "DataGetRequest" + ], + "response": "DataRowsResponse", + "statusCode": 200 + }, + "POST /twinkle/put": { + "body": [ + "DataPutRequest" + ], + "response": "DataRef", + "statusCode": 200 + }, + "POST /twinkle/release": { + "body": [ + "DataReleaseRequest" + ], + "response": "dict", + "statusCode": 200 + } + }, + "gateway": { + "DELETE /training_runs/{run_id}/checkpoints/{checkpoint_id}": { + "body": [], + "response": "Any", + "statusCode": 200 + }, + "DELETE /twinkle/training_runs/{run_id}/checkpoints/{checkpoint_id}": { + "body": [], + "response": "DeleteCheckpointResponse", + "statusCode": 200 + }, + "GET /get_server_capabilities": { + "body": [], + "response": "GetServerCapabilitiesResponse", + "statusCode": 200 + }, + "GET /healthz": { + "body": [], + "response": "HealthResponse", + "statusCode": 200 + }, + "GET /models": { + "body": [], + "response": null, + "statusCode": 200 + }, + "GET /training_runs": { + "body": [], + "response": "TrainingRunsResponse", + "statusCode": 200 + }, + "GET /training_runs/{run_id}": { + "body": [], + "response": "TrainingRun", + "statusCode": 200 + }, + "GET /training_runs/{run_id}/checkpoints": { + "body": [], + "response": "CheckpointsListResponse", + "statusCode": 200 + }, + "GET /twinkle/capacity_info": { + "body": [], + "response": "CapacityInfoResponse", + "statusCode": 200 + }, + "GET /twinkle/checkpoint_path/{run_id}/{checkpoint_id}": { + "body": [], + "response": "CheckpointPathResponse", + "statusCode": 200 + }, + "GET /twinkle/get_server_capabilities": { + "body": [], + "response": "GetServerCapabilitiesResponse", + "statusCode": 200 + }, + "GET /twinkle/healthz": { + "body": [], + "response": "HealthResponse", + "statusCode": 200 + }, + "GET /twinkle/healthz/deep": { + "body": [], + "response": "dict", + "statusCode": 200 + }, + "GET /twinkle/status": { + "body": [], + "response": "dict", + "statusCode": 200 + }, + "GET /twinkle/training_runs": { + "body": [], + "response": "TrainingRunsResponse", + "statusCode": 200 + }, + "GET /twinkle/training_runs/{run_id}": { + "body": [], + "response": "TrainingRun", + "statusCode": 200 + }, + "GET /twinkle/training_runs/{run_id}/checkpoints": { + "body": [], + "response": "CheckpointsListResponse", + "statusCode": 200 + }, + "POST /asample": { + "body": [ + "SampleRequest" + ], + "response": "Any", + "statusCode": 200 + }, + "POST /chat/completions": { + "body": [], + "response": null, + "statusCode": 200 + }, + "POST /create_model": { + "body": [ + "CreateModelRequest" + ], + "response": "Any", + "statusCode": 200 + }, + "POST /create_sampling_session": { + "body": [ + "CreateSamplingSessionRequest" + ], + "response": "CreateSamplingSessionResponse", + "statusCode": 200 + }, + "POST /create_session": { + "body": [ + "CreateSessionRequest" + ], + "response": "CreateSessionResponse", + "statusCode": 200 + }, + "POST /forward": { + "body": [ + "ForwardRequest" + ], + "response": "Any", + "statusCode": 200 + }, + "POST /forward_backward": { + "body": [ + "ForwardBackwardRequest" + ], + "response": "Any", + "statusCode": 200 + }, + "POST /get_info": { + "body": [ + "GetInfoRequest" + ], + "response": "Any", + "statusCode": 200 + }, + "POST /load_weights": { + "body": [ + "LoadWeightsRequest" + ], + "response": "Any", + "statusCode": 200 + }, + "POST /optim_step": { + "body": [ + "OptimStepRequest" + ], + "response": "Any", + "statusCode": 200 + }, + "POST /retrieve_future": { + "body": [ + "FutureRetrieveRequest" + ], + "response": "Any", + "statusCode": 200 + }, + "POST /save_weights": { + "body": [ + "SaveWeightsRequest" + ], + "response": "Any", + "statusCode": 200 + }, + "POST /save_weights_for_sampler": { + "body": [ + "SaveWeightsForSamplerRequest" + ], + "response": "Any", + "statusCode": 200 + }, + "POST /session_heartbeat": { + "body": [ + "SessionHeartbeatRequest" + ], + "response": "SessionHeartbeatResponse", + "statusCode": 200 + }, + "POST /telemetry": { + "body": [ + "TelemetrySendRequest" + ], + "response": "TelemetryResponse", + "statusCode": 200 + }, + "POST /training_runs/{run_id}/checkpoints/{checkpoint_id}/publish": { + "body": [], + "response": null, + "statusCode": 200 + }, + "POST /twinkle/cancel": { + "body": [ + "CancelRequest" + ], + "response": "CancelResponse", + "statusCode": 200 + }, + "POST /twinkle/create_session": { + "body": [ + "CreateSessionRequest" + ], + "response": "CreateSessionResponse", + "statusCode": 200 + }, + "POST /twinkle/retrieve_future": { + "body": [ + "RetrieveFutureRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/session_heartbeat": { + "body": [ + "SessionHeartbeatRequest" + ], + "response": "SessionHeartbeatResponse", + "statusCode": 200 + }, + "POST /twinkle/weights_info": { + "body": [ + "WeightsInfoRequest" + ], + "response": "WeightsInfoResponse", + "statusCode": 200 + }, + "POST /unload_model": { + "body": [ + "UnloadModelRequest" + ], + "response": "Any", + "statusCode": 200 + }, + "POST /weights_info": { + "body": [ + "dict" + ], + "response": "WeightsInfoResponse", + "statusCode": 200 + } + }, + "model": { + "GET /healthz": { + "body": [], + "response": "dict", + "statusCode": 200 + }, + "POST /tinker/create_model": { + "body": [ + "CreateModelRequest" + ], + "response": "UntypedAPIFuture", + "statusCode": 200 + }, + "POST /tinker/forward": { + "body": [ + "ForwardRequest" + ], + "response": "UntypedAPIFuture", + "statusCode": 200 + }, + "POST /tinker/forward_backward": { + "body": [ + "ForwardBackwardRequest" + ], + "response": "UntypedAPIFuture", + "statusCode": 200 + }, + "POST /tinker/get_info": { + "body": [ + "GetInfoRequest" + ], + "response": "GetInfoResponse", + "statusCode": 200 + }, + "POST /tinker/load_weights": { + "body": [ + "LoadWeightsRequest" + ], + "response": "UntypedAPIFuture", + "statusCode": 200 + }, + "POST /tinker/optim_step": { + "body": [ + "OptimStepRequest" + ], + "response": "UntypedAPIFuture", + "statusCode": 200 + }, + "POST /tinker/save_weights": { + "body": [ + "SaveWeightsRequest" + ], + "response": "UntypedAPIFuture", + "statusCode": 200 + }, + "POST /tinker/save_weights_for_sampler": { + "body": [ + "SaveWeightsForSamplerRequest" + ], + "response": "UntypedAPIFuture", + "statusCode": 200 + }, + "POST /tinker/unload_model": { + "body": [ + "UnloadModelRequest" + ], + "response": "UntypedAPIFuture", + "statusCode": 200 + }, + "POST /twinkle/add_adapter_to_model": { + "body": [ + "AddAdapterRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/add_metric": { + "body": [ + "AddMetricRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/apply_patch": { + "body": [ + "ApplyPatchRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/backward": { + "body": [ + "AdapterRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/calculate_loss": { + "body": [ + "AdapterRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/calculate_metric": { + "body": [ + "CalculateMetricRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/clip_grad_and_step": { + "body": [ + "ClipGradAndStepRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/clip_grad_norm": { + "body": [ + "AdapterRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/create": { + "body": [ + "CreateRequest" + ], + "response": "CreateResponse", + "statusCode": 200 + }, + "POST /twinkle/forward": { + "body": [ + "ForwardRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/forward_backward": { + "body": [ + "ForwardRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/forward_backward_from_data_plane": { + "body": [ + "DataPlaneForwardRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/forward_from_data_plane": { + "body": [ + "DataPlaneForwardRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/forward_only": { + "body": [ + "ForwardOnlyRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/forward_only_from_data_plane": { + "body": [ + "DataPlaneForwardOnlyRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/get_train_configs": { + "body": [ + "AdapterRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/load": { + "body": [ + "LoadRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/lr_step": { + "body": [ + "AdapterRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/remove_adapter": { + "body": [ + "AdapterRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/resume_from_checkpoint": { + "body": [ + "ResumeFromCheckpointRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/save": { + "body": [ + "SaveRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/set_loss": { + "body": [ + "SetLossRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/set_lr_scheduler": { + "body": [ + "SetLrSchedulerRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/set_optimizer": { + "body": [ + "SetOptimizerRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/set_processor": { + "body": [ + "SetProcessorRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/set_template": { + "body": [ + "SetTemplateRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/step": { + "body": [ + "AdapterRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/upload_to_hub": { + "body": [ + "UploadToHubRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/zero_grad": { + "body": [ + "AdapterRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + } + }, + "processor": { + "POST /twinkle/call": { + "body": [ + "ProcessorCallRequest" + ], + "response": "ProcessorCallResponse", + "statusCode": 200 + }, + "POST /twinkle/create": { + "body": [ + "ProcessorCreateRequest" + ], + "response": "ProcessorCreateResponse", + "statusCode": 200 + } + }, + "sampler": { + "POST /tinker/asample": { + "body": [ + "SampleRequest" + ], + "response": "UntypedAPIFuture", + "statusCode": 200 + }, + "POST /twinkle/add_adapter_to_sampler": { + "body": [ + "AddAdapterRequest" + ], + "response": "AddAdapterResponse", + "statusCode": 200 + }, + "POST /twinkle/apply_patch": { + "body": [ + "ApplyPatchRequest" + ], + "response": null, + "statusCode": 200 + }, + "POST /twinkle/create": { + "body": [], + "response": "CreateResponse", + "statusCode": 200 + }, + "POST /twinkle/sample": { + "body": [ + "SampleRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/sample_stream": { + "body": [ + "SampleRequest" + ], + "response": null, + "statusCode": 200 + }, + "POST /twinkle/sample_to_data_plane": { + "body": [ + "DataPlaneSampleRequest" + ], + "response": "TaskEnvelope", + "statusCode": 200 + }, + "POST /twinkle/set_template": { + "body": [ + "SetTemplateRequest" + ], + "response": "SetTemplateResponse", + "statusCode": 200 + }, + "POST /twinkle/unload_adapter_paths": { + "body": [ + "UnloadAdapterPathsRequest" + ], + "response": "dict", + "statusCode": 200 + } + } +} diff --git a/tests/server/contract/test_client_api_contract.py b/tests/server/contract/test_client_api_contract.py index ea0f743b..15e6cfb6 100644 --- a/tests/server/contract/test_client_api_contract.py +++ b/tests/server/contract/test_client_api_contract.py @@ -1,31 +1,63 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Zero-wire-change contract guard (T8.1 / R8 / Property 10). +"""Client-API wire-surface guards. -Exports the request/response surface of all five apps and compares it field-by-field with -the canonical baseline. The diff must be empty. Also asserts the load-bearing -invariants: ``schedule_task_and_wait`` still -exists and the only client-side additions are ``types/base.py`` and ``types/errors.py``. +Two guards with different strengths, kept apart on purpose: + +1. :func:`test_route_inventory_matches_committed_snapshot` compares the live route + inventory against ``client_api_routes.json``, which **is** committed. This is the + real guard: an unintended route addition/removal, or a changed response/body model, + fails here and shows up as a readable diff in the PR. +2. :func:`test_full_surface_extraction_is_self_consistent` only exercises the + field-level extractor. ``client_api_baseline.json`` is a generated, gitignored + artifact, so comparing against it cannot detect drift -- it would be comparing the + code to itself. The test is therefore scoped to what it can honestly assert: that + extraction runs, covers all five apps, and round-trips through JSON. + +Plus the load-bearing structural invariants of the request-lifecycle refactor. """ from __future__ import annotations +import json + import pytest -from tests.server.contract.client_api_harness import extract_full_surface, load_baseline +from tests.server.contract.client_api_harness import (extract_full_surface, extract_route_inventory, + load_route_inventory) + +_APPS = {'data_plane', 'gateway', 'model', 'processor', 'sampler'} + + +def test_route_inventory_matches_committed_snapshot(): + current = extract_route_inventory() + committed = load_route_inventory() + assert set(current) == _APPS + + diffs = [] + for app in sorted(set(current) | set(committed)): + cur_routes, old_routes = current.get(app, {}), committed.get(app, {}) + for key in sorted(set(cur_routes) | set(old_routes)): + if cur_routes.get(key) != old_routes.get(key): + diffs.append(f' [{app}] {key}: committed={old_routes.get(key)} current={cur_routes.get(key)}') + assert not diffs, ('Client-facing route surface differs from the committed inventory.\n' + 'If the change is intentional, regenerate and review the diff:\n' + ' python -m tests.server.contract.update_baseline\n' + '\n'.join(diffs)) -def test_wire_surface_matches_baseline(): - current = extract_full_surface() - baseline = load_baseline() - assert set(current) == {'data_plane', 'gateway', 'model', 'processor', 'sampler'} - assert current == baseline, ( - 'Client-facing wire surface changed vs the canonical baseline; ' - 'this spec must be zero-wire-change. Diffing apps: ' - f'{[a for a in set(current) | set(baseline) if current.get(a) != baseline.get(a)]}') +def test_full_surface_extraction_is_self_consistent(): + # Scoped to what a self-generated snapshot can prove: the extractor works. + surface = extract_full_surface() + assert set(surface) == _APPS + for app, contract in surface.items(): + assert contract['paths'], f'{app} exposed no routes' + assert json.loads(json.dumps(surface, sort_keys=True)) == surface -def test_schedule_task_and_wait_not_removed(): +def test_schedule_task_and_wait_removed(): + # server-request-lifecycle (Part 2) deletes the in-process blocking wait: the + # future record is now the single delivery channel (Property 1). from twinkle.server.utils.task_queue.mixin import TaskQueueMixin - assert hasattr(TaskQueueMixin, 'schedule_task_and_wait') + assert not hasattr(TaskQueueMixin, 'schedule_task_and_wait') + assert hasattr(TaskQueueMixin, 'submit_and_peek') def test_new_client_types_importable(): diff --git a/tests/server/contract/update_baseline.py b/tests/server/contract/update_baseline.py index 45609bfb..c819bff6 100644 --- a/tests/server/contract/update_baseline.py +++ b/tests/server/contract/update_baseline.py @@ -1,21 +1,30 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Regenerate the client-API contract baseline. +"""Regenerate the client-API contract snapshots. Run with:: python -m tests.server.contract.update_baseline -Only invoke after confirming that the current client-facing surface has been -intentionally changed and approved as part of this refactor. +Two artifacts, with deliberately different git treatment: + +- ``client_api_baseline.json`` -- the full field-level surface. NOT committed + (gitignored): an 8k-line diff on every intentional wire change is noise nobody + reads, and being regenerated from the code under test it proves nothing on its own. +- ``client_api_routes.json`` -- the compact route inventory. COMMITTED, and it is the + actual regression guard. Review its diff: every added/removed route and every + changed response/body model appears there as a readable line. + +Only regenerate after confirming that the current client-facing surface changed +intentionally. """ from __future__ import annotations -from tests.server.contract.client_api_harness import write_baseline +from tests.server.contract.client_api_harness import write_baseline, write_route_inventory def main() -> None: - path = write_baseline() - print(f'Wrote baseline: {path}') + print(f'Wrote generated baseline (not committed): {write_baseline()}') + print(f'Wrote route inventory (COMMIT THIS): {write_route_inventory()}') if __name__ == '__main__': diff --git a/tests/server/data_plane/test_store.py b/tests/server/data_plane/test_store.py index 3c7ec163..1bb001e6 100644 --- a/tests/server/data_plane/test_store.py +++ b/tests/server/data_plane/test_store.py @@ -8,7 +8,11 @@ @pytest.mark.asyncio async def test_data_ref_round_trip_append_release_and_ref_isolation(monkeypatch) -> None: - import transfer_queue as tq + # TransferQueue ships in the `async-rl` extra, not in `server` / `client`, so an + # environment installed without that extra must SKIP here rather than fail. A bare + # import turned an absent optional dependency into a permanently red test, which + # teaches readers to ignore red. + tq = pytest.importorskip('transfer_queue') records = {} diff --git a/tests/server/fixtures/server_config_mock.yaml b/tests/server/fixtures/server_config_mock.yaml index 39833f1c..67303ed5 100644 --- a/tests/server/fixtures/server_config_mock.yaml +++ b/tests/server/fixtures/server_config_mock.yaml @@ -12,10 +12,9 @@ http_options: persistence: # Gateway and Model run as separate Ray Serve replicas (separate - # processes); the tinker future flow needs cross-process visibility. - # ``memory`` mode is per-process; file (or redis) is required. - mode: file - file_path: /tmp/twinkle_state_mock.json + # processes); the tinker future flow needs cross-process visibility, which + # ``memory`` provides via a detached Ray named actor. + mode: memory applications: diff --git a/tests/server/integration/test_mock_mode_startup.py b/tests/server/integration/test_mock_mode_startup.py index be8610ce..f1838b2c 100644 --- a/tests/server/integration/test_mock_mode_startup.py +++ b/tests/server/integration/test_mock_mode_startup.py @@ -254,8 +254,6 @@ def _exercise_twinkle_clients(base: str) -> None: assert isinstance(metric.result, dict) cfgs = model.get_train_configs() assert isinstance(cfgs.result, str) - state = model.get_state_dict() - assert isinstance(state.result, dict) save_resp = model.save(name='step-1') assert save_resp.twinkle_path and save_resp.twinkle_path.startswith('twinkle://') diff --git a/tests/server/lifecycle/__init__.py b/tests/server/lifecycle/__init__.py new file mode 100644 index 00000000..85b3e739 --- /dev/null +++ b/tests/server/lifecycle/__init__.py @@ -0,0 +1 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. diff --git a/tests/server/lifecycle/test_envelope.py b/tests/server/lifecycle/test_envelope.py new file mode 100644 index 00000000..f7ff6321 --- /dev/null +++ b/tests/server/lifecycle/test_envelope.py @@ -0,0 +1,52 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Unit tests for the single FutureRecord -> TaskEnvelope mapping point (T1.2).""" +from __future__ import annotations + +from twinkle.server.lifecycle.envelope import envelope_from_record + + +def test_completed_with_none_result_is_a_success_not_a_failure(): + """R1#4 / Property 4: `completed` + `result is None` is a valid success.""" + env = envelope_from_record('req-1', {'status': 'completed', 'result': None}) + assert env.status == 'completed' + assert env.result is None + assert env.error is None + + +def test_completed_carries_result_and_no_error(): + env = envelope_from_record('req-1', {'status': 'completed', 'result': {'loss': 0.5}}) + assert env.result == {'loss': 0.5} + assert env.error is None + + +def test_legacy_two_field_failure_payload_is_backfilled_not_strict_validated(): + """R2#8 / Property 5: a pre-spec {error, category} payload maps to a legal envelope. + + It must go through error_payload_from_stored (missing error_code/request_id are + backfilled), never a strict ErrorPayload.model_validate that would raise. + """ + env = envelope_from_record('req-9', {'status': 'failed', 'result': {'error': 'boom', 'category': 'server'}}) + assert env.status == 'failed' + assert env.result is None + assert env.error is not None + assert env.error.error == 'boom' + assert env.error.category.value == 'server' + assert env.error.error_code == 500 # backfilled + assert env.error.request_id == 'req-9' # backfilled from the argument + + +def test_non_terminal_record_carries_queue_state_and_no_payload(): + env = envelope_from_record( + 'req-3', {'status': 'running', 'queue_state': 'active', 'queue_state_reason': 'x'}) + assert env.status == 'running' + assert env.result is None + assert env.error is None + assert env.queue_state == 'active' + assert env.queue_state_reason == 'x' + + +def test_missing_record_falls_back_to_pending(): + env = envelope_from_record('req-4', None) + assert env.status == 'pending' + assert env.result is None + assert env.error is None diff --git a/tests/server/lifecycle/test_envelope_coverage.py b/tests/server/lifecycle/test_envelope_coverage.py new file mode 100644 index 00000000..c6ece290 --- /dev/null +++ b/tests/server/lifecycle/test_envelope_coverage.py @@ -0,0 +1,66 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Task_Envelope coverage check (T5.4 / R5#7, R5#2/#3). + +Walks the model and sampler route tables and asserts that every twinkle-native +POST route that enters the Task_Queue declares ``response_model = TaskEnvelope``. +The exemption list (endpoints that do NOT enter the queue, plus the streaming +endpoint) is declared here, in one place. +""" +from __future__ import annotations + +from fastapi.routing import APIRoute + +from tests.server.contract.client_api_harness import build_model_app, build_sampler_app +from twinkle_client.types.lifecycle import TaskEnvelope + +# The single exemption declaration (R5#7), keyed BY APP. A flat path set would be wrong: +# ``/twinkle/set_template`` and ``/twinkle/apply_patch`` exist on both apps, but only the +# sampler's bypass the queue -- the model's are queued and must return a Task_Envelope. +# Sharing one set silently exempted the model's two and left a hole in this guard. +_EXEMPT_BY_APP = { + 'model': { + # health/session bootstrap only, no queue + '/twinkle/create', + }, + 'sampler': { + # direct call_backend, no queue + '/twinkle/create', + '/twinkle/set_template', + '/twinkle/add_adapter_to_sampler', + '/twinkle/apply_patch', + '/twinkle/unload_adapter_paths', + # the one streaming exception (R5#2) + '/twinkle/sample_stream', + }, +} + + +def _queued_twinkle_post_routes(app, exempt): + for route in app.routes: + if not isinstance(route, APIRoute): + continue + if 'POST' not in route.methods: + continue + if not route.path.startswith('/twinkle/'): + continue + if route.path in exempt: + continue + yield route + + +def test_every_queued_twinkle_route_returns_task_envelope(): + violations = [] + for app_name, app in (('model', build_model_app()), ('sampler', build_sampler_app())): + for route in _queued_twinkle_post_routes(app, _EXEMPT_BY_APP[app_name]): + if route.response_model is not TaskEnvelope: + violations.append((app_name, route.path, route.response_model)) + assert violations == [], f'queued routes not returning TaskEnvelope: {violations}' + + +def test_model_side_set_template_and_apply_patch_are_not_exempt(): + # Regression guard for the hole above: these two are queued on the model app, so + # they must be covered by the assertion rather than skipped by a shared path set. + assert '/twinkle/set_template' not in _EXEMPT_BY_APP['model'] + assert '/twinkle/apply_patch' not in _EXEMPT_BY_APP['model'] + covered = {route.path for route in _queued_twinkle_post_routes(build_model_app(), _EXEMPT_BY_APP['model'])} + assert {'/twinkle/set_template', '/twinkle/apply_patch'} <= covered diff --git a/tests/server/lifecycle/test_preflight_rejection.py b/tests/server/lifecycle/test_preflight_rejection.py new file mode 100644 index 00000000..4b17e9e6 --- /dev/null +++ b/tests/server/lifecycle/test_preflight_rejection.py @@ -0,0 +1,115 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Decision_Boundary tests: preflight rejects with real status codes and zero writes. + +Covers Property 3 / R8#3 (a rejected request writes no future record) and the +TwinkleServerError handler wire shape (R3#5/#6). No Ray or GPU is involved: the +task queue is driven with a spy state that counts ``store_future_status`` calls. +""" +from __future__ import annotations + +import pytest +from fastapi import FastAPI, Request +from fastapi.testclient import TestClient + +from twinkle.server.deployment import twinkle_server_error_handler +from twinkle.server.exceptions import (BatchSizeError, InputTokensExceededError, RateLimitExceededError, + RequestRejectedError, TwinkleServerError) +from twinkle.server.utils.task_queue.config import TaskQueueConfig +from twinkle.server.utils.task_queue.mixin import TaskQueueMixin + + +class _SpyState: + """Counts every future write so a rejected request can be proven to write nothing.""" + + def __init__(self): + self.store_calls = 0 + + async def store_future_status(self, *args, **kwargs): + self.store_calls += 1 + + async def get_future(self, request_id): + return None + + +class _Harness(TaskQueueMixin): + + def __init__(self, **config_kwargs): + self.state = _SpyState() + self.replica_id = 'test-replica' + self._init_task_queue(TaskQueueConfig(**config_kwargs), deployment_name='test') + + +async def _noop(): + return None + + +@pytest.mark.asyncio +async def test_input_tokens_rejection_is_422_and_zero_writes(): + """Property 3 / R8#3: an over-limit request raises 422 and writes no record.""" + h = _Harness(enabled=True, max_input_tokens=10) + try: + with pytest.raises(InputTokensExceededError) as exc: + await h.schedule_task(lambda: _noop(), model_id='m', token='tok', input_tokens=999, task_type='forward') + assert exc.value.error_code == 422 + assert exc.value.category.value == 'user' + assert h.state.store_calls == 0 + finally: + await h.shutdown_task_queue() + + +@pytest.mark.asyncio +async def test_batch_size_rejection_is_422_and_zero_writes(): + h = _Harness(enabled=True, max_input_tokens=100000) + try: + with pytest.raises(BatchSizeError): + await h.schedule_task( + lambda: _noop(), model_id='m', token='tok', input_tokens=1, + batch_size=1, data_world_size=4, task_type='forward') + assert h.state.store_calls == 0 + finally: + await h.shutdown_task_queue() + + +@pytest.mark.asyncio +async def test_rate_limit_rejection_is_429_and_zero_writes(): + h = _Harness(enabled=True, rps_limit=1, tps_limit=1000000, window_seconds=100, max_input_tokens=100000) + try: + # First call is admitted (it enqueues -> writes); reset the counter and + # assert the rejected second call (same window, rps=1) writes nothing. + await h.schedule_task(lambda: _noop(), model_id='m', token='tok', input_tokens=1, task_type='forward') + h.state.store_calls = 0 + with pytest.raises(RateLimitExceededError) as exc: + await h.schedule_task(lambda: _noop(), model_id='m', token='tok', input_tokens=1, task_type='forward') + assert exc.value.error_code == 429 + assert h.state.store_calls == 0 + finally: + await h.shutdown_task_queue() + + +@pytest.mark.asyncio +async def test_disabled_queue_skips_preflight(): + """The 'no token or queue disabled' short circuit is preserved (R3#3).""" + h = _Harness(enabled=False, max_input_tokens=10) + try: + ref = await h.schedule_task(lambda: _noop(), model_id='m', token='tok', input_tokens=999, task_type='forward') + assert 'request_id' in ref # not rejected: enqueued normally + finally: + await h.shutdown_task_queue() + + +def test_error_handler_puts_fields_at_top_level(): + """R3#5/#6: the handler returns error_code as the status and fields at top level.""" + app = FastAPI() + app.add_exception_handler(TwinkleServerError, twinkle_server_error_handler) + + @app.get('/boom') + async def boom(request: Request): + raise RequestRejectedError('nope', error_code=409) + + resp = TestClient(app, raise_server_exceptions=False).get('/boom') + assert resp.status_code == 409 + body = resp.json() + assert 'detail' not in body # not nested under detail + assert body['error'] == 'nope' + assert body['category'] == 'user' + assert body['error_code'] == 409 diff --git a/tests/server/lifecycle/test_retrieve_endpoint.py b/tests/server/lifecycle/test_retrieve_endpoint.py new file mode 100644 index 00000000..f2f78033 --- /dev/null +++ b/tests/server/lifecycle/test_retrieve_endpoint.py @@ -0,0 +1,86 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Wire tests for the twinkle Retrieve_Endpoint (T1.4, Property 4/5, R8#4/#5). + +These use a fake state and FastAPI's TestClient; no Ray runtime is needed, so they +live outside the state-actor fixtures. +""" +from __future__ import annotations + +import time + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from twinkle.server.gateway.twinkle_handlers import _register_twinkle_routes + + +class _State: + """A fake ServerState whose get_future returns a fixed record (or None).""" + + def __init__(self, record): + self._record = record + + async def get_future(self, request_id: str): + return self._record + + +class _Gateway: + + def __init__(self, record): + self.state = _State(record) + + +def _client(record) -> TestClient: + app = FastAPI() + _register_twinkle_routes(app, lambda: _Gateway(record)) + return TestClient(app) + + +def test_completed_with_null_result_returns_200_and_null(monkeypatch): + """Property 4 / R8#4: completed + result=None is 200 with result null, not 500.""" + client = _client({'status': 'completed', 'result': None}) + resp = client.post('/twinkle/retrieve_future', json={'request_id': 'req-1'}) + assert resp.status_code == 200 + body = resp.json() + assert body['status'] == 'completed' + assert body['result'] is None + assert body['error'] is None + + +def test_legacy_two_field_failure_returns_200_and_valid_envelope(): + """Property 5 / R8#5: a {error, category} record is 200 with a legal envelope.""" + client = _client({'status': 'failed', 'result': {'error': 'boom', 'category': 'server'}}) + resp = client.post('/twinkle/retrieve_future', json={'request_id': 'req-2'}) + assert resp.status_code == 200 + body = resp.json() + assert body['status'] == 'failed' + assert body['result'] is None + assert body['error']['error'] == 'boom' + assert body['error']['category'] == 'server' + assert body['error']['error_code'] == 500 + assert body['error']['request_id'] == 'req-2' + + +def test_always_missing_record_404s_only_after_the_full_window(monkeypatch): + """R2#5: a request_id that never appears returns 404, and only after waiting a window.""" + monkeypatch.setenv('TWINKLE_LONG_POLL_TIMEOUT', '0.3') + client = _client(None) + start = time.monotonic() + resp = client.post('/twinkle/retrieve_future', json={'request_id': 'ghost'}) + waited = time.monotonic() - start + assert resp.status_code == 404 + assert 'ghost' in resp.json()['detail'] + # It must fold the missing record into the wait loop, not short-circuit. + assert waited >= 0.3 + + +def test_terminal_record_returns_immediately(monkeypatch): + """A record already terminal must not wait out the window.""" + monkeypatch.setenv('TWINKLE_LONG_POLL_TIMEOUT', '30') + client = _client({'status': 'completed', 'result': {'ok': True}}) + start = time.monotonic() + resp = client.post('/twinkle/retrieve_future', json={'request_id': 'req-5'}) + waited = time.monotonic() - start + assert resp.status_code == 200 + assert resp.json()['result'] == {'ok': True} + assert waited < 5.0 diff --git a/tests/server/lifecycle/test_run_submit_dedup.py b/tests/server/lifecycle/test_run_submit_dedup.py new file mode 100644 index 00000000..a8861484 --- /dev/null +++ b/tests/server/lifecycle/test_run_submit_dedup.py @@ -0,0 +1,102 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""run_submit seq_id dedup: the release-on-failure decision must be driven by whether +a future record exists (i.e. whether the task was enqueued), NOT by exception type. + +The load-bearing case: if submit_and_peek raises *after* the task was enqueued (e.g. a +transient state error inside the inline peek), the seq claim must be KEPT -- releasing +it would let a retry enqueue a duplicate, the exact double-apply the dedup prevents. +""" +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from twinkle.server.lifecycle.submit import run_submit + + +class _FakeState: + def __init__(self, record_after_claim): + self._record_after_claim = record_after_claim + self.claimed = {} + self.released = [] + + async def claim_seq(self, dedup_key, request_id, ttl): + # Unseen -> claim it and let the caller proceed (returns None). + self.claimed[dedup_key] = request_id + return None + + async def get_future(self, request_id): + # Simulates whether a PENDING/QUEUED record was written (task enqueued). + return self._record_after_claim + + async def release_seq(self, dedup_key): + self.released.append(dedup_key) + + +class _FakeManagement: + def __init__(self, record_after_claim): + self.state = _FakeState(record_after_claim) + self._task_queue_config = SimpleNamespace(effective_execution_timeout=60.0) + + async def _on_request_start(self, request): + return 'token' + + async def submit_and_peek(self, *args, **kwargs): + # Fail *after* the (simulated) enqueue -- e.g. a state blip during the peek. + raise RuntimeError('transient state error during peek') + + +def _request(): + return SimpleNamespace(state=SimpleNamespace(session_id='sess-1', request_id='rq-1')) + + +def _body(): + return SimpleNamespace(adapter_name='ad', seq_id=7) + + +async def _call(self, body, adapter_name, token): # pragma: no cover - never invoked + return {'ok': True} + + +@pytest.mark.asyncio +async def test_release_kept_when_task_already_enqueued(): + # A record exists (task enqueued) -> peek error must NOT release the claim. + mgmt = _FakeManagement(record_after_claim={'status': 'queued'}) + with pytest.raises(RuntimeError): + await run_submit(mgmt, _request(), _body(), task_type='forward_backward', backend_call=_call) + assert mgmt.state.released == [], 'claim wrongly released for an already-enqueued task' + assert 'seq::sess-1::sess-1-ad::7' in mgmt.state.claimed + + +@pytest.mark.asyncio +async def test_release_when_never_enqueued(): + # No record (e.g. preflight rejected before any write) -> release so a retry can re-enqueue. + mgmt = _FakeManagement(record_after_claim=None) + with pytest.raises(RuntimeError): + await run_submit(mgmt, _request(), _body(), task_type='forward_backward', backend_call=_call) + assert mgmt.state.released == ['seq::sess-1::sess-1-ad::7'], 'claim should be released when nothing enqueued' + + +@pytest.mark.asyncio +async def test_dedup_key_is_scoped_per_adapter(): + """Two adapters in ONE session must not collide on the same seq_id. + + Every client model object owns its own seq counter starting at 1 while ``session_id`` + is process-global, so multi-LoRA training from one process issues seq_id=1 twice. If + the adapter were missing from the key, the second adapter's forward_backward would be + swallowed as a duplicate and handed the first adapter's loss -- a silent wrong result. + """ + mgmt = _FakeManagement(record_after_claim={'status': 'queued'}) + for adapter in ('lora-A', 'lora-B'): + with pytest.raises(RuntimeError): + await run_submit( + mgmt, + _request(), + SimpleNamespace(adapter_name=adapter, seq_id=1), + task_type='forward_backward', + backend_call=_call) + + claimed = set(mgmt.state.claimed) + assert claimed == {'seq::sess-1::sess-1-lora-A::1', 'seq::sess-1::sess-1-lora-B::1'}, ( + f'adapters collided on one dedup key: {claimed}') diff --git a/tests/server/lifecycle/test_static_guards.py b/tests/server/lifecycle/test_static_guards.py new file mode 100644 index 00000000..5caef4bc --- /dev/null +++ b/tests/server/lifecycle/test_static_guards.py @@ -0,0 +1,111 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Static / structural guards for the lifecycle refactor. + +- Property 1 (R8#7): the deleted symbols occur zero times under ``src/twinkle/**``. +- Property 0: ``TaskEnvelope`` has exactly one construction site. +- D5 client-side invariant (R1#12): the client HTTP timeout is <= 120 and strictly + greater than the server Long_Poll_Window. +- The task status set has two independent declarations that must not drift. + +These exist because the spec states most of its guarantees in prose. A prose claim that +nothing checks decays into a false claim -- as happened with "a consistency test asserts +the two sets are equal", which was written in a docstring while no such test existed. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_SRC = _REPO_ROOT / 'src' / 'twinkle' + +# Symbols the refactor removed. A wildcard search (not a per-file list) must find +# each of them zero times across the whole server tree (R8#7, Property 1). +_FORBIDDEN_SYMBOLS = ( + 'schedule_task_and_wait', + 'run_task', + 'persist_status', + '_complete_result', + '_complete_error', +) + + +@pytest.mark.parametrize('symbol', _FORBIDDEN_SYMBOLS) +def test_deleted_symbol_has_zero_occurrences(symbol): + hits = [] + for path in _SRC.rglob('*.py'): + text = path.read_text(encoding='utf-8') + if symbol in text: + hits.append(str(path.relative_to(_REPO_ROOT))) + assert hits == [], f'{symbol!r} still occurs in: {hits}' + + +def test_client_http_timeout_bounds(): + from twinkle.server.lifecycle.poll_config import long_poll_window + from twinkle_client.http.http_utils import _HTTP_TIMEOUT + + assert _HTTP_TIMEOUT <= 120 + assert _HTTP_TIMEOUT > long_poll_window() + + +def test_task_envelope_has_exactly_one_construction_site(): + """Property 0's structural precondition: one mapping point, mechanically enforced. + + ``envelope_from_record`` is the only place a FutureRecord becomes a TaskEnvelope, so + ``failed`` always lands in ``error`` and never in ``result`` regardless of which + endpoint answered. That was previously only a docstring claim -- a handler building a + ``TaskEnvelope(...)`` itself would silently reintroduce the exact defect the single + mapping point exists to prevent (a failure inside the Inline_Fast_Path window losing + its payload), and every existing test would still pass. + """ + sites = [] + for path in _SRC.rglob('*.py'): + text = path.read_text(encoding='utf-8') + for lineno, line in enumerate(text.splitlines(), start=1): + if 'TaskEnvelope(' in line and 'class TaskEnvelope' not in line: + sites.append(f'{path.relative_to(_REPO_ROOT)}:{lineno}') + + offenders = [site for site in sites if 'server/lifecycle/envelope.py' not in site] + assert offenders == [], ('TaskEnvelope must only be constructed in lifecycle/envelope.py ' + f'(via envelope_from_record); found: {offenders}') + assert sites, 'expected to find the construction sites inside envelope.py' + + +def test_server_task_status_enum_matches_client_literal(): + """The two independent declarations of the task status set must not drift. + + ``twinkle_client.types.lifecycle.TaskStatus`` (a Literal on the wire model) and the + server's ``TaskStatus`` enum are declared separately. ``envelope_from_record`` copies + ``record['status']`` straight into ``TaskEnvelope.status``, so a value the server can + write but the Literal does not list would fail pydantic validation *while serialising + the response* -- i.e. a 500 from retrieve for a task that actually finished. + + The client module's comment claimed such a test existed; it did not. This is it. + """ + from typing import get_args + + from twinkle.server.utils.task_queue.types import TaskStatus as ServerTaskStatus + from twinkle_client.types.lifecycle import TaskStatus as WireTaskStatus, TERMINAL_STATUSES + + server_values = {member.value for member in ServerTaskStatus} + wire_values = set(get_args(WireTaskStatus)) + assert server_values == wire_values, (f'task status sets drifted: server-only={server_values - wire_values}, ' + f'wire-only={wire_values - server_values}') + assert TERMINAL_STATUSES <= wire_values, 'TERMINAL_STATUSES must be a subset of the declared statuses' + + +def test_client_future_layer_is_not_imported_by_the_server(): + """``_future.py`` carries an underscore because the dependency runs one way only. + + The server reverse-imports ``twinkle_client.types`` (the shared wire contract), but the + client's polling layer is private to the client. An import in the other direction would + make the server depend on client retry policy, which its own long-poll already owns. + Another claim that lived only in a docstring. + """ + offenders = [] + for path in _SRC.rglob('*.py'): + text = path.read_text(encoding='utf-8') + if 'twinkle_client._future' in text or 'from twinkle_client import _future' in text: + offenders.append(str(path.relative_to(_REPO_ROOT))) + assert offenders == [], f'server must not import the client future layer: {offenders}' diff --git a/tests/server/lifecycle/test_submit_peek_e2e.py b/tests/server/lifecycle/test_submit_peek_e2e.py new file mode 100644 index 00000000..0009a64c --- /dev/null +++ b/tests/server/lifecycle/test_submit_peek_e2e.py @@ -0,0 +1,83 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""End-to-end proof of the Inline_Fast_Path + Client_Future_Layer seam (T2.3). + +A minimal harness drives the *real* ``submit_and_peek`` against a real compute +worker and a real (memory) ServerState -- no HTTP, no GPU. The resulting envelope +is round-tripped through model_dump/model_validate (simulating the wire) and fed to +the real client ``resolve``, so this covers the exact submit -> client path that +Property 0 protects. +""" +from __future__ import annotations + +import pytest + +ray = pytest.importorskip('ray') + +from twinkle.server.state import ServerState # noqa: E402 +from twinkle.server.utils.task_queue.config import TaskQueueConfig # noqa: E402 +from twinkle.server.utils.task_queue.mixin import TaskQueueMixin # noqa: E402 +from twinkle_client import _future # noqa: E402 +from twinkle_client.exceptions import TaskFailedError # noqa: E402 +from twinkle_client.types.lifecycle import TaskEnvelope # noqa: E402 + + +class _Harness(TaskQueueMixin): + """Real task queue + real state, with a window generous enough to be deterministic.""" + + def __init__(self) -> None: + self.state = ServerState() + self.replica_id = 'test-replica' + # enabled=False skips rate limiting; a 5s window makes "task finishes inside + # the window" deterministic for a trivial in-process coroutine. + self._init_task_queue( + TaskQueueConfig(enabled=False, inline_fast_path_timeout=5.0), deployment_name='test') + + +def _across_the_wire(env: TaskEnvelope) -> TaskEnvelope: + return TaskEnvelope.model_validate(env.model_dump(mode='json')) + + +@pytest.mark.asyncio +async def test_window_completed_task_is_single_round_trip(monkeypatch): + """R8#1: a task terminal within the window makes the client issue zero retrieves.""" + h = _Harness() + + async def _ok(): + return None + + try: + env = await h.submit_and_peek(lambda: _ok(), task_type='step') + assert env.status == 'completed' + assert env.result is None + + monkeypatch.setattr(_future, '_post_retrieve', + lambda _r: pytest.fail('completed submit must not poll retrieve')) + assert _future.resolve(_across_the_wire(env), model_cls=None) is None + finally: + await h.shutdown_task_queue() + + +@pytest.mark.asyncio +async def test_window_failed_task_surfaces_payload_as_taskfailed(monkeypatch): + """R8#2 / Property 0: a failure inside the window reaches the client via the submit + response and is raised as TaskFailedError with its payload intact.""" + h = _Harness() + + async def _boom(): + raise ValueError('kaboom') + + try: + env = await h.submit_and_peek(lambda: _boom(), task_type='step') + assert env.status == 'failed' + assert env.error is not None + assert 'kaboom' in env.error.error + + monkeypatch.setattr(_future, '_post_retrieve', + lambda _r: pytest.fail('failed submit must not poll retrieve')) + with pytest.raises(TaskFailedError) as exc: + _future.resolve(_across_the_wire(env), model_cls=None) + assert 'kaboom' in exc.value.error + assert exc.value.category == 'server' + assert exc.value.error_code == 500 + finally: + await h.shutdown_task_queue() diff --git a/tests/server/lifecycle/test_timing_bounds.py b/tests/server/lifecycle/test_timing_bounds.py new file mode 100644 index 00000000..7d484977 --- /dev/null +++ b/tests/server/lifecycle/test_timing_bounds.py @@ -0,0 +1,139 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Timing guards for the lifecycle constants. + +- Property 2 (R1#6): a single Submit_Endpoint's server-side duration is bounded by the + Inline_Fast_Path window + 1s and is INDEPENDENT of how long the task itself runs. This + is the spec's core benefit claim and was previously the only property with no automated + guard. +- R2#4 / R2#6 / D4: the retrieve poll interval is a single shared declaration, is strictly + inside the Long_Poll_Window, and is deliberately FIXED (see the measurement recorded in + ``poll_config`` and in the test below). +""" +from __future__ import annotations + +import asyncio +import time +from pathlib import Path + +import pytest + +ray = pytest.importorskip('ray') + +from twinkle.server.lifecycle.poll_config import long_poll_window, retrieve_poll_interval # noqa: E402 +from twinkle.server.state import ServerState # noqa: E402 +from twinkle.server.utils.task_queue.config import TaskQueueConfig # noqa: E402 +from twinkle.server.utils.task_queue.mixin import TaskQueueMixin # noqa: E402 + +_WINDOW = 0.05 + + +class _Harness(TaskQueueMixin): + def __init__(self) -> None: + self.state = ServerState() + self.replica_id = 'test-replica' + self._init_task_queue( + TaskQueueConfig(enabled=False, inline_fast_path_timeout=_WINDOW), deployment_name='test') + + +@pytest.mark.asyncio +async def test_submit_duration_is_bounded_and_task_duration_independent(): + """Property 2: submit returns on the window, not on task completion.""" + h = _Harness() + + async def fast(): + return None + + async def slow(): + await asyncio.sleep(5.0) + return {'loss': 1.0} + + try: + # Warm up first: the very first submit pays for creating the detached state actor + # and starting the ComputeWorker (~10s on a cold Ray). That cost is not part of the + # submit path this test is about, and timing it made the test pass or fail depending + # on whether an earlier test in the directory had already warmed the backend. + await h.submit_and_peek(lambda: fast(), task_type='warmup') + + started = time.monotonic() + env = await h.submit_and_peek(lambda: slow(), task_type='forward_backward') + elapsed = time.monotonic() - started + + # Bounded by the window + 1s even though the task needs 5s (R1#6). + assert elapsed < _WINDOW + 1.0, f'submit took {elapsed:.3f}s, expected < {_WINDOW + 1.0}s' + # 5s task cannot have finished, so the envelope must be non-terminal. + assert env.status not in ('completed', 'failed', 'cancelled'), env.status + assert env.result is None and env.error is None + assert env.request_id + finally: + await h.shutdown_task_queue() + + +def test_poll_interval_satisfies_the_constant_chain(): + interval, window = retrieve_poll_interval(), long_poll_window() + assert interval > 0, 'a non-positive interval would busy-spin the state backend' + assert interval < window, 'a single poll step must not exceed the whole window' + + +def test_both_retrieve_endpoints_share_one_interval_declaration(): + """R2#6: one declaration point, and no endpoint reading os.environ on its own. + + Also pins the measured decision: a FIXED interval, not exponential backoff. The + backoff variant was implemented, measured on real PPU hardware, and reverted -- + control-plane ops (0.00-0.09s) never reach retrieve because the 50ms inline window + absorbs them, while ``forward_backward`` lands at 0.52-0.65s where a 0.05->1.0s + doubling schedule checks at 0.80 instead of the fixed schedule's 0.55, costing ~22% + per step. If backoff is ever reintroduced, it needs a ceiling around 0.2s and an + explicit decision to pay 2.5x the poll rate. + """ + import twinkle.server.gateway.tinker_handlers as tinker_h + import twinkle.server.gateway.twinkle_handlers as twinkle_h + from twinkle.server.lifecycle import poll_config + + assert not hasattr(poll_config, 'initial_poll_interval'), 'backoff was reverted by measurement' + assert not hasattr(poll_config, 'max_poll_interval'), 'backoff was reverted by measurement' + + for module in (tinker_h, twinkle_h): + source = Path(module.__file__).read_text(encoding='utf-8') + assert 'TWINKLE_POLL_INTERVAL' not in source, f'{module.__name__} reads the env var directly' + assert 'TWINKLE_LONG_POLL_TIMEOUT' not in source, f'{module.__name__} reads the env var directly' + + +def test_gateway_guard_warns_once_per_value_not_once_per_request(monkeypatch): + """D5 guard must be audible but not spam. + + ``long_poll_window()`` runs on the hot path of both retrieve endpoints, not only at + startup, so an unguarded warning would repeat on every retrieve request (~2/s during + training). It must fire for a misconfigured value, stay silent on repeats of that same + value, and speak up again when the value changes to a different offending one. + + Counts calls on a stub logger rather than using ``caplog``: ``get_logger()`` sets + ``propagate = False``, so records never reach the root handler caplog installs. + """ + from twinkle.server.lifecycle import poll_config + + class _Counter: + def __init__(self): + self.warnings = 0 + + def warning(self, *_args, **_kwargs): + self.warnings += 1 + + counter = _Counter() + monkeypatch.setattr(poll_config, 'logger', counter) + monkeypatch.setattr(poll_config, '_warned_window', None, raising=False) + + def warnings_while(value: str, calls: int) -> int: + monkeypatch.setenv('TWINKLE_LONG_POLL_TIMEOUT', value) + before = counter.warnings + for _ in range(calls): + assert poll_config.long_poll_window() == float(value) + return counter.warnings - before + + assert warnings_while('90', calls=5) == 1, 'an over-the-gateway-limit window must warn exactly once' + assert warnings_while('90', calls=5) == 0, 'repeats of the same value must stay silent' + assert warnings_while('120', calls=3) == 1, 'a different offending value must warn again' + assert warnings_while('30', calls=5) == 0, 'a safe window must never warn' + + +if __name__ == '__main__': + raise SystemExit(pytest.main([__file__, '-v'])) diff --git a/tests/server/lifecycle/test_tinker_retrieve_regression.py b/tests/server/lifecycle/test_tinker_retrieve_regression.py new file mode 100644 index 00000000..ab8d07b3 --- /dev/null +++ b/tests/server/lifecycle/test_tinker_retrieve_regression.py @@ -0,0 +1,60 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Tinker /retrieve_future wire regression (T6.3 / Property 6 / R8#8). + +The tinker endpoint's response shape and status-code semantics must be unchanged by +this spec, across all three shapes: ``try_again`` / ``{error, category}`` / bare +result. It shares the poll_config window but keeps its own wire contract. +""" +from __future__ import annotations + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from twinkle.server.gateway.tinker_handlers import _register_tinker_routes + + +class _State: + + def __init__(self, record): + self._record = record + + async def get_future(self, request_id): + return self._record + + +class _Gateway: + + def __init__(self, record): + self.state = _State(record) + + +def _client(record): + app = FastAPI() + _register_tinker_routes(app, lambda: _Gateway(record)) + return TestClient(app) + + +def test_try_again_shape_for_non_terminal(monkeypatch): + monkeypatch.setenv('TWINKLE_LONG_POLL_TIMEOUT', '0.2') + monkeypatch.setenv('TWINKLE_POLL_INTERVAL', '0.05') + resp = _client({'status': 'running', 'queue_state': 'active'}).post( + '/retrieve_future', json={'request_id': 'r'}) + assert resp.status_code == 200 + assert resp.json()['type'] == 'try_again' + + +def test_error_category_shape_for_failed(): + resp = _client({'status': 'failed', 'result': {'error': 'boom', 'category': 'server'}}).post( + '/retrieve_future', json={'request_id': 'r'}) + assert resp.status_code == 200 + body = resp.json() + assert body['error'] == 'boom' + assert body['category'] == 'server' + assert 'type' not in body + + +def test_bare_result_shape_for_completed(): + resp = _client({'status': 'completed', 'result': {'foo': 'bar'}}).post( + '/retrieve_future', json={'request_id': 'r'}) + assert resp.status_code == 200 + assert resp.json() == {'foo': 'bar'} diff --git a/tests/server/model/test_mock_model.py b/tests/server/model/test_mock_model.py index 5c24450f..5564d434 100644 --- a/tests/server/model/test_mock_model.py +++ b/tests/server/model/test_mock_model.py @@ -47,7 +47,6 @@ 'save', 'load', 'resume_from_checkpoint', - 'get_state_dict', 'get_train_configs', 'add_adapter', 'add_adapter_to_model', diff --git a/tests/server/model/test_twinkle_async_inputs.py b/tests/server/model/test_twinkle_async_inputs.py index 3221394a..22059725 100644 --- a/tests/server/model/test_twinkle_async_inputs.py +++ b/tests/server/model/test_twinkle_async_inputs.py @@ -59,9 +59,11 @@ async def get(self, ref, *, fields=None): return rows return [{field: row[field] for field in fields} for row in rows] - async def schedule_task_and_wait(self, task, **kwargs): - self.scheduled.append(kwargs) - return await task() + async def submit_and_peek(self, coro_factory, *, model_id=None, token=None, task_type=None, **schedule_kwargs): + self.scheduled.append(schedule_kwargs) + result = await coro_factory() + from twinkle_client.types.lifecycle import TaskEnvelope + return TaskEnvelope(request_id='req-test', status='completed', result=result) async def call_backend(self, fn, /, *args, admit=True, **kwargs): return fn(*args, **kwargs) diff --git a/tests/server/sampler/test_mock_sampler.py b/tests/server/sampler/test_mock_sampler.py index 8efae14b..f9a2806d 100644 --- a/tests/server/sampler/test_mock_sampler.py +++ b/tests/server/sampler/test_mock_sampler.py @@ -115,6 +115,12 @@ def test_mock_dispatch_returns_mock_sampler() -> None: def test_explicit_async_vllm_uses_non_blocking_sampler(monkeypatch) -> None: + # Nothing here needs TransferQueue, but importing it is unavoidable: the + # `twinkle_agentic.async_rl` package eagerly pulls in `native_tq`, which subclasses + # `transfer_queue.GRPOGroupNSampler` at module level. That optional dependency lives + # in the `async-rl` extra, so skip when it is absent instead of failing a sampler test + # for a data-path dependency it does not use. + pytest.importorskip('transfer_queue') from twinkle_agentic.async_rl import vllm_sampler_tq as module captured = {} diff --git a/tests/server/sampler/test_twinkle_async_rows.py b/tests/server/sampler/test_twinkle_async_rows.py index 234437be..b6a59df7 100644 --- a/tests/server/sampler/test_twinkle_async_rows.py +++ b/tests/server/sampler/test_twinkle_async_rows.py @@ -74,9 +74,11 @@ def __init__(self): async def _on_request_start(self, _request): return 'token' - async def schedule_task_and_wait(self, task, **kwargs): - self.scheduled.append(kwargs) - return await task() + async def submit_background_and_peek(self, coro_factory, *, model_id=None, task_type=None): + self.scheduled.append({'model_id': model_id, 'task_type': task_type}) + result = await coro_factory() + from twinkle_client.types.lifecycle import TaskEnvelope + return TaskEnvelope(request_id='req-test', status='completed', result=result) async def call_backend(self, fn, /, *args, admit=True, **kwargs): return fn(*args, **kwargs) @@ -130,14 +132,15 @@ async def test_sample_to_data_plane_returns_ref_after_short_admission() -> None: sampling_params={'max_tokens': 4}, ) - ref = await route.endpoint(request, body, management) + env = await route.endpoint(request, body, management) + ref = types.DataRef(**env.result) assert ref.ref_id == 'rollout-ref' + # The whole admit -> generate -> store runs as one background future now + # (vLLM owns generation concurrency), so there is a single scheduled task. assert management.scheduled == [{ 'model_id': 'session-adapter', - 'token': 'token', - 'input_tokens': 1, - 'task_type': 'sample_admission', + 'task_type': 'sample_to_data_plane', }] assert management.put_rows == [{ 'train_input': {'input_ids': [1, 7], 'labels': [-100, 7]}, diff --git a/tests/server/state/fake_backend.py b/tests/server/state/fake_backend.py new file mode 100644 index 00000000..247cd662 --- /dev/null +++ b/tests/server/state/fake_backend.py @@ -0,0 +1,76 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""In-process ``StateBackend`` test double. + +Both shipped backends need infrastructure -- ``memory`` starts a detached Ray actor, +``redis`` needs a server -- so state-logic tests that only care about +``FutureManager`` / manager semantics use this dict-backed fake instead. Running on +a single event loop, ``set_nx`` / ``update_atomic`` are atomic simply by not +awaiting between read and write, mirroring the real backends' guarantee. +""" +from __future__ import annotations + +import time +from collections.abc import Callable +from fnmatch import fnmatch +from typing import Any + +from twinkle.server.state.backend.base import StateBackend + + +class FakeBackend(StateBackend): + """Dict-backed backend with TTL support, for tests only.""" + + def __init__(self) -> None: + self._store: dict[str, tuple[Any, float | None]] = {} + + def _is_expired(self, key: str) -> bool: + entry = self._store.get(key) + if entry is None: + return True + _, expire_at = entry + if expire_at is not None and time.time() >= expire_at: + del self._store[key] + return True + return False + + async def set(self, key: str, value: Any, ttl: int | None = None) -> None: + self._store[key] = (value, (time.time() + ttl) if ttl is not None else None) + + async def get(self, key: str) -> Any | None: + if self._is_expired(key): + return None + return self._store[key][0] + + async def delete(self, key: str) -> None: + self._store.pop(key, None) + + async def exists(self, key: str) -> bool: + return not self._is_expired(key) + + async def keys(self, pattern: str) -> list[str]: + return [k for k in list(self._store) if not self._is_expired(k) and fnmatch(k, pattern)] + + async def count(self, pattern: str) -> int: + return len(await self.keys(pattern)) + + async def set_nx(self, key: str, value: Any, ttl: int | None = None) -> bool: + if not self._is_expired(key): + return False + await self.set(key, value, ttl) + return True + + async def update_atomic( + self, + key: str, + transform: Callable[[Any | None], Any | None], + ttl: int | None = None, + ) -> Any | None: + current = await self.get(key) + updated = transform(current) + if updated is None: + return current + await self.set(key, updated, ttl) + return updated + + async def close(self) -> None: + self._store.clear() diff --git a/tests/server/state/test_future_lifecycle.py b/tests/server/state/test_future_lifecycle.py index 55905afa..d423dcfe 100644 --- a/tests/server/state/test_future_lifecycle.py +++ b/tests/server/state/test_future_lifecycle.py @@ -1,23 +1,88 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """State-hygiene tests for FutureManager cleanup and the do-not-regress guard. -Spec: T5.6 / R9#5 / R9#6 / Property 6 / Property 7. Uses the Ray-free FileBackend. +Spec: T5.6 / R9#5 / R9#6 / Property 6 / Property 7. Both shipped backends need +infrastructure (``memory`` starts a detached Ray actor, ``redis`` needs a server), +so these pure ``FutureManager`` semantics run against the dict-backed fake below. """ from __future__ import annotations import time +from collections.abc import Callable +from fnmatch import fnmatch +from typing import Any from unittest import mock import pytest +from twinkle.server.state.backend.base import StateBackend from twinkle.server.state.future_manager import FutureManager +class _FakeBackend(StateBackend): + """Dict-backed StateBackend. Single event loop, so ``update_atomic`` is atomic + simply by not awaiting between read and write -- the real backends' guarantee.""" + + def __init__(self) -> None: + self._store: dict[str, tuple[Any, float | None]] = {} + + def _is_expired(self, key: str) -> bool: + entry = self._store.get(key) + if entry is None: + return True + _, expire_at = entry + if expire_at is not None and time.time() >= expire_at: + del self._store[key] + return True + return False + + async def set(self, key: str, value: Any, ttl: int | None = None) -> None: + self._store[key] = (value, (time.time() + ttl) if ttl is not None else None) + + async def get(self, key: str) -> Any | None: + return None if self._is_expired(key) else self._store[key][0] + + async def delete(self, key: str) -> None: + self._store.pop(key, None) + + async def exists(self, key: str) -> bool: + return not self._is_expired(key) + + async def keys(self, pattern: str) -> list[str]: + return [k for k in list(self._store) if not self._is_expired(k) and fnmatch(k, pattern)] + + async def count(self, pattern: str) -> int: + return len(await self.keys(pattern)) + + async def set_nx(self, key: str, value: Any, ttl: int | None = None) -> bool: + if not self._is_expired(key): + return False + await self.set(key, value, ttl) + return True + + async def update_atomic( + self, + key: str, + transform: Callable[[Any | None], Any | None], + ttl: int | None = None, + ) -> Any | None: + current = await self.get(key) + updated = transform(current) + if updated is None: + return current + await self.set(key, updated, ttl) + return updated + + async def close(self) -> None: + self._store.clear() + + async def health_check(self) -> bool: + return True + + @pytest.fixture -def manager(tmp_path): - from twinkle.server.state.backend.file_backend import FileBackend - backend = FileBackend(str(tmp_path / 'state.json')) - return FutureManager(backend, expiration_timeout=300.0) +def manager(): + return FutureManager(_FakeBackend(), expiration_timeout=300.0) async def _store(manager, request_id, status, *, replica_id=None, absolute_deadline=None): @@ -120,3 +185,43 @@ async def test_stored_timestamps_align_with_wall_clock_regardless_of_host_tz(man rec = await manager.get('r8') parsed = manager._parse_timestamp(rec.created_at) assert before - 1 <= parsed <= after + 1 + + +@pytest.mark.asyncio +async def test_claim_seq_dedups_then_release_readmits(): + from twinkle.server.state.server_state import ServerState + state = ServerState(backend=_FakeBackend()) + # First claim of a (session, seq_id) is unseen -> None, caller proceeds to enqueue. + assert await state.claim_seq('seq::s1::1', 'reqA', ttl=60) is None + # A duplicate claim returns the original request_id -> caller returns its envelope. + assert await state.claim_seq('seq::s1::1', 'reqB', ttl=60) == 'reqA' + # Releasing (e.g. the original was preflight-rejected) re-admits the same seq_id. + await state.release_seq('seq::s1::1') + assert await state.claim_seq('seq::s1::1', 'reqC', ttl=60) is None + # Different session with the same seq_id never collides. + assert await state.claim_seq('seq::s2::1', 'reqD', ttl=60) is None + + +@pytest.mark.asyncio +async def test_cancel_drops_pending_but_never_running(): + from twinkle.server.state.server_state import ServerState + state = ServerState(backend=_FakeBackend()) + # pending -> cancel drops it to the terminal 'cancelled' state with a user payload. + await state.store_future_status('rp', 'pending', 'm1') + assert await state.cancel_future('rp') == {'cancelled': True, 'state': 'cancelled'} + rec = await state.get_future('rp') + assert rec['status'] == 'cancelled' and rec['result']['error_code'] == 499 + # running -> cancel is a no-op; in-flight work is never interrupted. + await state.store_future_status('rr', 'running', 'm1') + assert await state.cancel_future('rr') == {'cancelled': False, 'state': 'running'} + # unknown request_id -> not_found. + assert await state.cancel_future('nope') == {'cancelled': False, 'state': 'not_found'} + + +def test_cancelled_record_maps_to_error_envelope(): + from twinkle.server.lifecycle.envelope import envelope_from_record + rec = {'status': 'cancelled', + 'result': {'error': 'Task cancelled by client', 'category': 'user', 'error_code': 499}} + env = envelope_from_record('rc', rec) + assert env.status == 'cancelled' + assert env.error is not None and env.error.error_code == 499 diff --git a/tests/server/state/test_update_atomic.py b/tests/server/state/test_update_atomic.py index 012e7f23..5c7ca7be 100644 --- a/tests/server/state/test_update_atomic.py +++ b/tests/server/state/test_update_atomic.py @@ -18,11 +18,9 @@ import os import pytest import pytest_asyncio -import tempfile import uuid from typing import Any -from twinkle.server.state.backend.file_backend import FileBackend from twinkle.server.state.backend.memory_backend import RayActorBackend REDIS_URL = os.environ.get('TWINKLE_TEST_REDIS_URL', 'redis://localhost:6379/0') @@ -71,13 +69,6 @@ def _replace_with(current: Any | None, *, value: Any) -> Any: return value -def _file_backend() -> FileBackend: - f = tempfile.NamedTemporaryFile(suffix='.json', delete=False) - f.close() - os.unlink(f.name) - return FileBackend(f.name) - - def _redis_backend(): from twinkle.server.state.backend.redis_backend import RedisBackend @@ -89,12 +80,6 @@ def memory_backend() -> RayActorBackend: return RayActorBackend() -@pytest.fixture -def file_backend(): - backend = _file_backend() - yield backend - - @pytest_asyncio.fixture async def redis_backend(): backend = _redis_backend() @@ -158,40 +143,6 @@ async def test_memory_set_nx_with_ttl(memory_backend) -> None: assert await memory_backend.set_nx('lease', 'next', ttl=1) is True -# ---------- File backend ------------------------------------------------- # - - -@pytest.mark.asyncio -async def test_file_update_atomic_read_transform_write(file_backend) -> None: - await file_backend.set('k', 5) - result = await file_backend.update_atomic('k', functools.partial(_increment_or_init, delta=3)) - assert result == 8 - assert await file_backend.get('k') == 8 - - -@pytest.mark.asyncio -async def test_file_update_atomic_none_is_noop(file_backend) -> None: - await file_backend.set('k', 42) - result = await file_backend.update_atomic('k', _no_op) - assert result == 42 - - -@pytest.mark.asyncio -async def test_file_update_atomic_respects_ttl(file_backend) -> None: - await file_backend.update_atomic('leased', functools.partial(_replace_with, value='holder'), ttl=1) - assert await file_backend.get('leased') == 'holder' - await asyncio.sleep(1.1) - assert await file_backend.get('leased') is None - - -@pytest.mark.asyncio -async def test_file_set_nx_with_ttl(file_backend) -> None: - assert await file_backend.set_nx('lease', 'owner', ttl=1) is True - assert await file_backend.set_nx('lease', 'other', ttl=1) is False - await asyncio.sleep(1.1) - assert await file_backend.set_nx('lease', 'next', ttl=1) is True - - # ---------- Redis backend ------------------------------------------------ # diff --git a/tests/server/utils/test_task_queue_mixin.py b/tests/server/utils/test_task_queue_mixin.py index 04200285..74447e5e 100644 --- a/tests/server/utils/test_task_queue_mixin.py +++ b/tests/server/utils/test_task_queue_mixin.py @@ -12,9 +12,20 @@ class _DummyState: def __init__(self): self.records = [] + self._latest = {} async def store_future_status(self, *args, **kwargs): self.records.append((args, kwargs)) + request_id, status = args[0], args[1] + self._latest[request_id] = { + 'status': status, + 'result': kwargs.get('result'), + 'queue_state': kwargs.get('queue_state'), + 'queue_state_reason': kwargs.get('queue_state_reason'), + } + + async def get_future(self, request_id): + return self._latest.get(request_id) class _AllowingRateLimiter: @@ -27,7 +38,9 @@ class _DummyQueue(TaskQueueMixin): def __init__(self): self.state = _DummyState() - self._task_queue_config = TaskQueueConfig() + # A generous Inline_Fast_Path window keeps "task settles inside submit" + # deterministic for the trivial in-process coroutines used here. + self._task_queue_config = TaskQueueConfig(inline_fast_path_timeout=5.0) self._rate_limiter = _AllowingRateLimiter() self._task_metrics = None self._deployment_name = 'test' @@ -45,21 +58,20 @@ def enable_compute_worker(self): @pytest.mark.asyncio async def test_preflight_rejects_batch_without_per_dp_multiple(): queue = _DummyQueue() + from twinkle.server.exceptions import BatchSizeError - result = await queue._perform_preflight_checks( - request_id='req1', - model_id='model1', - token='token1', - input_tokens=0, - batch_size=2, - data_world_size=2, - batch_size_multiple=2, - ) + with pytest.raises(BatchSizeError, match='must be divisible by 4'): + await queue._perform_preflight_checks( + model_id='model1', + token='token1', + input_tokens=0, + batch_size=2, + data_world_size=2, + batch_size_multiple=2, + ) - assert result == {'request_id': 'req1', 'model_id': 'model1'} - _, kwargs = queue.state.records[-1] - assert kwargs['result']['category'] == 'user' - assert 'Batch size 2 must be divisible by 4' in kwargs['result']['error'] + # Property 3: a rejection writes no future record. + assert queue.state.records == [] @pytest.mark.asyncio @@ -67,7 +79,6 @@ async def test_preflight_accepts_batch_with_per_dp_multiple(): queue = _DummyQueue() result = await queue._perform_preflight_checks( - request_id='req1', model_id='model1', token='token1', input_tokens=0, @@ -99,8 +110,9 @@ async def work(): @pytest.mark.asyncio -async def test_schedule_task_and_wait_returns_large_result_without_persisting_it(): +async def test_submit_and_peek_returns_completed_envelope_and_persists(): queue = _DummyQueue() + queue.replica_id = 'replica-1' queue.enable_compute_worker() result = {'logps': [[float(index) for index in range(128)]]} @@ -108,7 +120,7 @@ async def work(): return result try: - actual = await queue.schedule_task_and_wait( + env = await queue.submit_and_peek( work, model_id='model1', token='token1', @@ -117,8 +129,10 @@ async def work(): finally: await queue._compute_worker.stop() - assert actual is result - assert queue.state.records == [] + assert env.status == 'completed' + assert env.result == result + # The future record is now the single delivery channel: the result IS persisted. + assert any(args[1] == 'completed' for args, _ in queue.state.records) @pytest.mark.asyncio @@ -152,25 +166,28 @@ async def work(): @pytest.mark.asyncio -async def test_schedule_task_and_wait_propagates_failure_without_persisting_it(): +async def test_submit_and_peek_failure_returns_failed_envelope_and_persists(): queue = _DummyQueue() + queue.replica_id = 'replica-1' queue.enable_compute_worker() async def work(): raise ValueError('model failed') try: - with pytest.raises(RuntimeError, match='ValueError: model failed'): - await queue.schedule_task_and_wait( - work, - model_id='model1', - token='token1', - task_type='forward_backward', - ) + env = await queue.submit_and_peek( + work, + model_id='model1', + token='token1', + task_type='forward_backward', + ) finally: await queue._compute_worker.stop() - assert queue.state.records == [] + # Property 0: the failure payload rides the envelope's `error` field. + assert env.status == 'failed' + assert env.error is not None and 'model failed' in env.error.error + assert any(args[1] == 'failed' for args, _ in queue.state.records) @pytest.mark.asyncio @@ -196,15 +213,16 @@ async def work(): @pytest.mark.asyncio -async def test_schedule_task_and_wait_reports_preflight_failure_without_persisting_it(): +async def test_submit_and_peek_preflight_rejection_raises_without_writing_or_queuing(): + from twinkle.server.exceptions import BatchSizeError queue = _DummyQueue() queue.enable_compute_worker() async def work(): raise AssertionError('preflight rejection must not execute the task') - with pytest.raises(RuntimeError, match='Batch size 2 must be divisible by 4'): - await queue.schedule_task_and_wait( + with pytest.raises(BatchSizeError, match='must be divisible by 4'): + await queue.submit_and_peek( work, model_id='model1', token='token1', @@ -215,3 +233,19 @@ async def work(): assert queue.state.records == [] assert queue._compute_worker._worker_task is None + + +@pytest.mark.asyncio +async def test_submit_and_peek_honors_explicit_request_id(): + # run_submit generates the request_id up front (to claim the seq dedup key + # atomically) and threads it through submit_and_peek; the future record and the + # returned envelope must use that exact id, not a freshly generated one. + queue = _DummyQueue() + queue.enable_compute_worker() + + async def _factory(): + return {'ok': True} + + env = await queue.submit_and_peek(_factory, task_type='step', request_id='req_fixed123') + assert env.request_id == 'req_fixed123' + assert env.status == 'completed' diff --git a/tests/twinkle_client/test_async_components.py b/tests/twinkle_client/test_async_components.py index 99b91fcc..55689637 100644 --- a/tests/twinkle_client/test_async_components.py +++ b/tests/twinkle_client/test_async_components.py @@ -20,6 +20,11 @@ def json(self): return self._payload +def _completed(result): + """Wrap a business result in a completed Task_Envelope (the new wire shape).""" + return {'request_id': 'req-test', 'status': 'completed', 'result': result} + + def test_model_forward_backward_sends_multiple_data_refs(monkeypatch) -> None: import twinkle_client.http as http_module from twinkle_client.model import multi_lora_transformers as module @@ -30,7 +35,7 @@ def post(*, url, json_data=None, **_kwargs): calls.append((url, json_data)) if url.endswith('/create'): return _Response({}) - return _Response({'result': {'loss': 1.0}}) + return _Response(_completed({'result': {'loss': 1.0}})) monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') monkeypatch.setattr(module, 'http_post', post) @@ -63,7 +68,7 @@ def test_model_inline_forward_methods_keep_the_original_endpoints(monkeypatch) - def post(*, url, json_data=None, **_kwargs): calls.append((url, json_data)) - return _Response({} if url.endswith('/create') else {'result': {}}) + return _Response({} if url.endswith('/create') else _completed({'result': {}})) monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') monkeypatch.setattr(module, 'http_post', post) @@ -92,7 +97,7 @@ def test_model_data_plane_forward_uses_a_separate_api(monkeypatch) -> None: def post(url, json_data=None, **_kwargs): calls.append((url, json_data)) - return _Response({} if url.endswith('/create') else {'result': {'value': 1}}) + return _Response({} if url.endswith('/create') else _completed({'result': {'value': 1}})) monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') monkeypatch.setattr(module, 'http_post', post) @@ -117,7 +122,7 @@ def test_model_data_plane_forward_only_can_append_selected_outputs(monkeypatch) def post(*, url, json_data=None, **_kwargs): calls.append((url, json_data)) - return _Response({} if url.endswith('/create') else {'result': updated_ref.model_dump()}) + return _Response({} if url.endswith('/create') else _completed({'result': updated_ref.model_dump()})) monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') monkeypatch.setattr(module, 'http_post', post) @@ -153,7 +158,7 @@ def post(*, url, json_data=None, **_kwargs): calls.append((url, json_data)) if url.endswith('/create'): return _Response({}) - return _Response(output_ref.model_dump()) + return _Response(_completed(output_ref.model_dump())) monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') monkeypatch.setattr(module, 'http_post', post) diff --git a/tests/twinkle_client/test_error_parsing.py b/tests/twinkle_client/test_error_parsing.py new file mode 100644 index 00000000..7a63dc61 --- /dev/null +++ b/tests/twinkle_client/test_error_parsing.py @@ -0,0 +1,69 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Client error-response parsing (T3.4 / Requirement 3 #7-#11).""" +from __future__ import annotations + +import pytest +import requests + +from twinkle_client.exceptions import TwinkleHTTPError +from twinkle_client.http.http_utils import _handle_response + + +class _Resp: + """Minimal stand-in for requests.Response for _handle_response.""" + + def __init__(self, status_code, *, body=None, text='', url='http://x'): + self.status_code = status_code + self.ok = status_code < 400 + self._body = body + self.text = text + self.url = url + + def json(self): + if self._body is None: + raise ValueError('no json') + return self._body + + +def test_structured_error_reads_top_level_fields(): + """R3#7/#8: top-level category/error_code/request_id are preferred.""" + resp = _Resp(422, body={'error': 'bad input', 'category': 'user', 'error_code': 422, 'request_id': 'req-7'}) + with pytest.raises(TwinkleHTTPError) as exc: + _handle_response(resp) + assert isinstance(exc.value, requests.HTTPError) # R3#8: existing except clauses keep working + assert exc.value.status_code == 422 + assert exc.value.error_code == 422 + assert exc.value.category == 'user' + assert exc.value.request_id == 'req-7' + assert 'bad input' in str(exc.value) + + +def test_detail_only_error_falls_back_to_unknown_category(): + """R3#7: FastAPI's built-in {detail: ...} maps to category='Unknown'.""" + resp = _Resp(404, body={'detail': 'Not Found'}) + with pytest.raises(TwinkleHTTPError) as exc: + _handle_response(resp) + assert exc.value.status_code == 404 + assert exc.value.category == 'Unknown' + assert exc.value.error_code is None + assert 'Not Found' in str(exc.value) + + +def test_non_json_body_falls_back_to_text(): + resp = _Resp(500, body=None, text='raw traceback text') + with pytest.raises(TwinkleHTTPError) as exc: + _handle_response(resp) + assert exc.value.category == 'Unknown' + assert 'raw traceback text' in str(exc.value) + + +def test_410_raises_stop_iteration_not_http_error(): + """R3#9: 410 keeps raising StopIteration, not an HTTP error.""" + resp = _Resp(410, body={'detail': 'exhausted'}) + with pytest.raises(StopIteration): + _handle_response(resp) + + +def test_ok_response_passes_through(): + resp = _Resp(200, body={'status': 'ok'}) + assert _handle_response(resp) is resp diff --git a/tests/twinkle_client/test_future_layer.py b/tests/twinkle_client/test_future_layer.py new file mode 100644 index 00000000..db0ff132 --- /dev/null +++ b/tests/twinkle_client/test_future_layer.py @@ -0,0 +1,137 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Client_Future_Layer unit tests (T2.2 / Requirement 4). + +``resolve`` is exercised against fabricated envelopes and a monkeypatched +``_post_retrieve``; no server or network is involved. +""" +from __future__ import annotations + +import pytest +import requests + +from twinkle_client import _future +from twinkle_client.exceptions import TaskFailedError, TaskRecordLostError, TaskWaitTimeoutError +from twinkle_client.types.errors import ErrorPayload +from twinkle_client.types.lifecycle import TaskEnvelope + + +class _Model: + """A model_cls that records what it deserialized.""" + + def __init__(self, result): + self.result = result + + @classmethod + def model_validate(cls, value): + return _Model(value) + + +def _completed(result): + return TaskEnvelope(request_id='r', status='completed', result=result) + + +def _failed(**kw): + payload = ErrorPayload(error='boom', category='server', error_code=500, request_id='r', **kw) + return TaskEnvelope(request_id='r', status='failed', error=payload) + + +def _running(): + return TaskEnvelope(request_id='r', status='running', queue_state='active') + + +def test_terminal_submit_issues_no_retrieve(monkeypatch): + """R8#1: a task terminal in the submit envelope makes zero retrieve calls.""" + def _boom(_request_id): + raise AssertionError('retrieve must not be called for a terminal submit') + + monkeypatch.setattr(_future, '_post_retrieve', _boom) + out = _future.resolve(_completed({'loss': 1.0}), model_cls=_Model) + assert out.result == {'loss': 1.0} + + +def test_terminal_submit_failure_raises_taskfailed_with_payload(monkeypatch): + """Property 0: a failure in the submit envelope raises TaskFailedError, payload intact.""" + monkeypatch.setattr(_future, '_post_retrieve', lambda _r: pytest.fail('no retrieve')) + with pytest.raises(TaskFailedError) as exc: + _future.resolve(_failed(), model_cls=_Model) + assert exc.value.error == 'boom' + assert exc.value.category == 'server' + assert exc.value.request_id == 'r' + assert exc.value.error_code == 500 + assert not isinstance(exc.value, requests.HTTPError) # R3#10 + + +def test_model_cls_none_returns_none_result(monkeypatch): + """R4#9: a method that returned None before still returns None (not swallowed).""" + monkeypatch.setattr(_future, '_post_retrieve', lambda _r: pytest.fail('no retrieve')) + assert _future.resolve(_completed(None), model_cls=None) is None + + +def test_non_terminal_submit_polls_until_terminal(monkeypatch): + replies = [_running(), _running(), _completed({'ok': 1})] + monkeypatch.setattr(_future, '_post_retrieve', lambda _r: replies.pop(0)) + out = _future.resolve(_running(), model_cls=_Model) + assert out.result == {'ok': 1} + assert replies == [] + + +def test_404_is_bounded_then_raises_record_lost(monkeypatch): + def _always_404(_request_id): + e = requests.HTTPError('404') + e.status_code = 404 + raise e + + monkeypatch.setattr(_future, '_post_retrieve', _always_404) + with pytest.raises(TaskRecordLostError): + _future.resolve(_running(), model_cls=_Model) + + +def test_transport_5xx_is_bounded_then_reraises(monkeypatch): + monkeypatch.setattr(_future.time, 'sleep', lambda _s: None) # no real backoff sleeps + + def _always_503(_request_id): + e = requests.HTTPError('503') + e.status_code = 503 + raise e + + monkeypatch.setattr(_future, '_post_retrieve', _always_503) + with pytest.raises(requests.HTTPError): + _future.resolve(_running(), model_cls=_Model) + + +def test_non_retryable_4xx_reraises_immediately(monkeypatch): + def _400(_request_id): + e = requests.HTTPError('400') + e.status_code = 400 + raise e + + monkeypatch.setattr(_future, '_post_retrieve', _400) + with pytest.raises(requests.HTTPError): + _future.resolve(_running(), model_cls=_Model) + + +def test_total_timeout_raises_wait_timeout(monkeypatch): + monkeypatch.setattr(_future, '_post_retrieve', lambda _r: _running()) + with pytest.raises(TaskWaitTimeoutError) as exc: + _future.resolve(_running(), model_cls=_Model, total_timeout=0.0) + assert exc.value.request_id == 'r' + + +def test_success_resets_both_retry_counters(monkeypatch): + """R4#8: a successful reply zeroes both counters, so intermittent 404s never sum up.""" + seq = [] + + def _mixed(_request_id): + seq.append(1) + n = len(seq) + if n in (1, 2, 4, 5): # 404s interleaved with a success at n==3 + e = requests.HTTPError('404') + e.status_code = 404 + raise e + if n == 3: + return _running() # success resets not_found_count + return _completed({'done': True}) + + monkeypatch.setattr(_future, '_post_retrieve', _mixed) + out = _future.resolve(_running(), model_cls=_Model) + assert out.result == {'done': True} From d848a5ac61cd6de3e72807a491988641cc9615a6 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Thu, 17 Sep 2026 16:34:42 +0000 Subject: [PATCH 17/32] refactor(server): extract sampler seams and rename session-resource mixins - move sampler weight resolution to sampler/weights.py and streaming bridge to sampler/backends/streaming.py so handlers stay thin - rename utils/lifecycle to utils/session_resource: the package holds session-scoped resource mixins, not the request lifecycle owned by twinkle/server/lifecycle - add static guards for adapter-name mapping and package-root imports - declare grimp test dependency used by the import-boundary guards --- pyproject.toml | 3 +- .../server/checkpoint/checkpoint_manager.py | 7 -- src/twinkle/server/checkpoint/tinker.py | 11 +-- src/twinkle/server/checkpoint/twinkle.py | 9 +-- src/twinkle/server/config/application_spec.py | 63 +++++++++++++++- src/twinkle/server/deployment.py | 21 ++++-- src/twinkle/server/exceptions.py | 13 ++++ src/twinkle/server/gateway/proxy.py | 16 ++++- .../server/launcher/env_propagation.py | 12 ++++ .../server/launcher/server_launcher.py | 17 +++++ src/twinkle/server/lifecycle/submit.py | 21 +++++- src/twinkle/server/model/app.py | 8 +-- src/twinkle/server/model/twinkle_handlers.py | 16 ++++- src/twinkle/server/processor/app.py | 2 +- .../server/sampler/backends/__init__.py | 22 +++--- .../server/sampler/backends/mock_sampler.py | 2 +- .../server/sampler/backends/streaming.py | 26 +++++++ src/twinkle/server/sampler/tinker_handlers.py | 20 +++--- .../server/sampler/twinkle_handlers.py | 70 +++++------------- src/twinkle/server/sampler/weights.py | 35 +++++++++ src/twinkle/server/state/server_state.py | 41 +++++++++-- src/twinkle/server/telemetry/middleware.py | 2 +- src/twinkle/server/telemetry/worker_init.py | 5 +- src/twinkle/server/utils/__init__.py | 2 +- .../server/utils/lifecycle/__init__.py | 8 --- .../server/utils/session_resource/__init__.py | 14 ++++ .../adapter.py | 34 +++------ .../{lifecycle => session_resource}/base.py | 25 ++++++- .../processor.py | 27 +++---- src/twinkle/server/utils/task_queue/mixin.py | 12 +++- src/twinkle/server/utils/task_queue/worker.py | 21 ++++++ tests/server/config/server_config_4b_e2e.yaml | 1 + .../config/server_config_4b_e2e_megatron.yaml | 1 + tests/server/config/test_server_config.py | 28 +++++++- tests/server/integration/test_sft_e2e.py | 61 +++++++++++++++- .../lifecycle/test_to_backend_inputs.py | 48 +++++++++++++ .../sampler/test_resolve_sampler_weights.py | 53 ++++++++++++++ tests/server/start_e2e_server.py | 12 ++-- .../server/static/backend_call_exemptions.py | 17 +++-- .../static/test_adapter_name_mapping.py | 71 +++++++++++++++++++ .../static/test_no_direct_backend_call.py | 63 +++++++++++++++- .../static/test_no_package_root_imports.py | 61 ++++++++++++++++ .../test_deployment_exception_boundary.py | 10 ++- tests/server/utils/test_task_queue_mixin.py | 35 +++++++-- 44 files changed, 840 insertions(+), 206 deletions(-) create mode 100644 src/twinkle/server/sampler/backends/streaming.py create mode 100644 src/twinkle/server/sampler/weights.py delete mode 100644 src/twinkle/server/utils/lifecycle/__init__.py create mode 100644 src/twinkle/server/utils/session_resource/__init__.py rename src/twinkle/server/utils/{lifecycle => session_resource}/adapter.py (75%) rename src/twinkle/server/utils/{lifecycle => session_resource}/base.py (90%) rename src/twinkle/server/utils/{lifecycle => session_resource}/processor.py (81%) create mode 100644 tests/server/lifecycle/test_to_backend_inputs.py create mode 100644 tests/server/sampler/test_resolve_sampler_weights.py create mode 100644 tests/server/static/test_adapter_name_mapping.py create mode 100644 tests/server/static/test_no_package_root_imports.py diff --git a/pyproject.toml b/pyproject.toml index 4b52c9c9..68500f29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,8 @@ server = [ test = [ "hypothesis>=6.0", "pytest", - "pytest-asyncio" + "pytest-asyncio", + "grimp>=3.0" ] docs = [ "sphinx>=5.3.0,<6.0.0", diff --git a/src/twinkle/server/checkpoint/checkpoint_manager.py b/src/twinkle/server/checkpoint/checkpoint_manager.py index 038bc309..538ea97b 100644 --- a/src/twinkle/server/checkpoint/checkpoint_manager.py +++ b/src/twinkle/server/checkpoint/checkpoint_manager.py @@ -29,7 +29,6 @@ class BaseCheckpointManager(BaseFileManager, ABC): Subclasses must implement: - path_prefix property - - path_field_name property - _create_checkpoint method - _parse_checkpoint method - _create_checkpoints_response method @@ -54,12 +53,6 @@ def path_prefix(self) -> str: """Return the path prefix (e.g., 'twinkle://').""" pass - @property - @abstractmethod - def path_field_name(self) -> str: - """Return the field name for the path (e.g., 'twinkle_path' or 'tinker_path').""" - pass - @abstractmethod def _create_checkpoint(self, checkpoint_id: str, diff --git a/src/twinkle/server/checkpoint/tinker.py b/src/twinkle/server/checkpoint/tinker.py index 518a60d7..ac72e46e 100644 --- a/src/twinkle/server/checkpoint/tinker.py +++ b/src/twinkle/server/checkpoint/tinker.py @@ -8,7 +8,9 @@ from tinker import types as tinker_types from typing import Any, Dict, List, Optional -from twinkle.server.checkpoint import TRAIN_RUN_INFO_FILENAME, BaseCheckpointManager, BaseTrainingRunManager +from twinkle.server.checkpoint.checkpoint_manager import BaseCheckpointManager +from twinkle.server.checkpoint.paths import TRAIN_RUN_INFO_FILENAME +from twinkle.server.checkpoint.training_run_manager import BaseTrainingRunManager class TinkerTrainingRunManager(BaseTrainingRunManager): @@ -73,10 +75,6 @@ class TinkerCheckpointManager(BaseCheckpointManager): def path_prefix(self) -> str: return 'twinkle://' - @property - def path_field_name(self) -> str: - return 'tinker_path' - def _create_checkpoint(self, checkpoint_id, checkpoint_type, @@ -130,6 +128,3 @@ def _create_parsed_path(self, path, training_run_id, checkpoint_type, def _create_weights_info(self, run_info: dict[str, Any]) -> tinker_types.WeightsInfoResponse: return tinker_types.WeightsInfoResponse(**run_info) - - def parse_tinker_path(self, tinker_path: str) -> tinker_types.ParsedCheckpointTinkerPath | None: - return self.parse_path(tinker_path) diff --git a/src/twinkle/server/checkpoint/twinkle.py b/src/twinkle/server/checkpoint/twinkle.py index c9036c6a..78eb2745 100644 --- a/src/twinkle/server/checkpoint/twinkle.py +++ b/src/twinkle/server/checkpoint/twinkle.py @@ -7,8 +7,9 @@ from datetime import datetime from typing import Any, Dict, List, Optional -from twinkle.server.checkpoint import (TRAIN_RUN_INFO_FILENAME, BaseCheckpointManager, BaseTrainingRunManager, - validate_ownership) +from twinkle.server.checkpoint.checkpoint_manager import BaseCheckpointManager +from twinkle.server.checkpoint.paths import TRAIN_RUN_INFO_FILENAME, validate_ownership +from twinkle.server.checkpoint.training_run_manager import BaseTrainingRunManager from twinkle_client.types.training import (Checkpoint, CheckpointsListResponse, CreateModelRequest, Cursor, ParsedCheckpointTwinklePath, TrainingRun, TrainingRunsResponse, WeightsInfoResponse) @@ -64,10 +65,6 @@ class TwinkleCheckpointManager(BaseCheckpointManager): def path_prefix(self) -> str: return 'twinkle://' - @property - def path_field_name(self) -> str: - return 'twinkle_path' - def _create_checkpoint(self, checkpoint_id, checkpoint_type, diff --git a/src/twinkle/server/config/application_spec.py b/src/twinkle/server/config/application_spec.py index 5e6fa89b..60792d5d 100644 --- a/src/twinkle/server/config/application_spec.py +++ b/src/twinkle/server/config/application_spec.py @@ -11,11 +11,24 @@ """ from __future__ import annotations +import os from pydantic import BaseModel, ConfigDict, Field, model_validator from typing import Any, Literal from twinkle.server.utils.task_queue.config import TaskQueueConfig +# Env var keys the launcher sets from the gateway ``server_config`` so that any +# Ray worker (model / sampler / processor), not just the gateway, applies the +# configured ServerState policy instead of ``get_server_state``'s hardcoded +# defaults. Only the four *policy* fields are propagated; ``actor_name`` is a +# per-process cache key, not a cross-worker policy, so it is excluded. +SERVER_STATE_ENV_KEYS: tuple[str, ...] = ( + 'TWINKLE_SERVER_STATE_EXPIRATION_TIMEOUT', + 'TWINKLE_SERVER_STATE_CLEANUP_INTERVAL', + 'TWINKLE_SERVER_STATE_PER_TOKEN_MODEL_LIMIT', + 'TWINKLE_SERVER_STATE_METRICS_UPDATE_INTERVAL', +) + # ---------- shared helpers ------------------------------------------------- # @@ -95,6 +108,47 @@ class ServerStateArgs(_ArgsBase): metrics_update_interval: float | None = None actor_name: str | None = None + def to_env_vars(self) -> dict[str, str]: + """Serialize the policy fields to env vars for Ray-worker propagation. + + Mirrors :meth:`PersistenceConfig.to_env_vars`. Only the four policy + fields are emitted (``actor_name`` is a per-process cache key, not a + policy); unset (``None``) fields are skipped so the worker falls back to + ``ServerState``'s own defaults. + """ + env: dict[str, str] = {} + if self.expiration_timeout is not None: + env['TWINKLE_SERVER_STATE_EXPIRATION_TIMEOUT'] = str(self.expiration_timeout) + if self.cleanup_interval is not None: + env['TWINKLE_SERVER_STATE_CLEANUP_INTERVAL'] = str(self.cleanup_interval) + if self.per_token_model_limit is not None: + env['TWINKLE_SERVER_STATE_PER_TOKEN_MODEL_LIMIT'] = str(self.per_token_model_limit) + if self.metrics_update_interval is not None: + env['TWINKLE_SERVER_STATE_METRICS_UPDATE_INTERVAL'] = str(self.metrics_update_interval) + return env + + @classmethod + def from_env(cls) -> ServerStateArgs | None: + """Reconstruct policy fields from launcher-set env vars. + + Returns ``None`` when no ``TWINKLE_SERVER_STATE_*`` key is set, so a + caller can distinguish "no env-configured policy" from "explicitly + configured to a default value". Only policy fields are populated; + ``actor_name`` stays ``None``. + """ + exp = os.environ.get('TWINKLE_SERVER_STATE_EXPIRATION_TIMEOUT') + clean = os.environ.get('TWINKLE_SERVER_STATE_CLEANUP_INTERVAL') + limit = os.environ.get('TWINKLE_SERVER_STATE_PER_TOKEN_MODEL_LIMIT') + interval = os.environ.get('TWINKLE_SERVER_STATE_METRICS_UPDATE_INTERVAL') + if exp is None and clean is None and limit is None and interval is None: + return None + return cls( + expiration_timeout=float(exp) if exp is not None else None, + cleanup_interval=float(clean) if clean is not None else None, + per_token_model_limit=int(limit) if limit is not None else None, + metrics_update_interval=float(interval) if interval is not None else None, + ) + class ServerArgs(_ArgsBase): """Args for the gateway ``server`` deployment.""" @@ -106,12 +160,17 @@ class ServerArgs(_ArgsBase): class ProcessorArgs(_ArgsBase): - """Args for the ``processor`` deployment.""" + """Args for the ``processor`` deployment. + + A processor deployment has no task queue, so there is deliberately no + ``queue_config`` field here: with ``extra='forbid'`` a YAML that sets + ``processor.args.queue_config`` now fails validation with the offending path + instead of being silently ignored. + """ ncpu_proc_per_node: int | None = None device_group: dict[str, Any] | None = None device_mesh: dict[str, Any] | None = None - queue_config: TaskQueueConfig = Field(default_factory=TaskQueueConfig) class DataPlaneArgs(_ArgsBase): diff --git a/src/twinkle/server/deployment.py b/src/twinkle/server/deployment.py index cc67ae0e..bf02ce76 100644 --- a/src/twinkle/server/deployment.py +++ b/src/twinkle/server/deployment.py @@ -40,7 +40,7 @@ from twinkle.server.telemetry.tracing import create_tracing_middleware from twinkle.server.utils.validation import verify_request_token from twinkle.utils.logger import get_logger -from twinkle_client.types.errors import ErrorPayload +from twinkle_client.types.errors import ErrorCategory, ErrorPayload logger = get_logger() @@ -164,10 +164,21 @@ async def ensure_state_cleanup_started(request: Request, call_next): async def catch_unhandled_exceptions(request: Request, call_next): try: return await call_next(request) - except Exception: - error = traceback.format_exc() - logger.error(error) - return JSONResponse(status_code=500, content={'detail': error}) + except Exception as exc: + tb = traceback.format_exc() + logger.error(tb) + # Unify the last-resort 500 with the rest of the wire: an + # ``ErrorPayload`` body (Server category keeps the traceback) instead + # of the legacy ``{'detail': }`` shape. + request_id = getattr(request.state, 'request_id', None) or '' + payload = ErrorPayload( + error=(str(exc) or exc.__class__.__name__), + category=ErrorCategory.Server, + error_code=500, + request_id=request_id, + traceback=tb, + ) + return JSONResponse(status_code=500, content=payload.model_dump(mode='json', exclude_none=True)) @app.middleware('http') async def verify_token(request: Request, call_next): diff --git a/src/twinkle/server/exceptions.py b/src/twinkle/server/exceptions.py index 1a3ba656..afe197d4 100644 --- a/src/twinkle/server/exceptions.py +++ b/src/twinkle/server/exceptions.py @@ -133,6 +133,19 @@ class RateLimitExceededError(RequestRejectedError): category = ErrorCategory.User +class ResourceNotFoundError(RequestRejectedError): + """A well-formed request names a resource (adapter / session) that is absent. + + Distinct from a malformed request (400): the request itself is valid but the + referenced resource does not exist or is expiring, so it is a 404 on the + Decision_Boundary left. Raised (never ``assert``-ed) so the check survives + ``python -O`` and is classified as user-facing rather than a 500. + """ + + error_code = 404 + category = ErrorCategory.User + + class FullModeBusyError(RequestRejectedError): """A full-parameter (exclusive) model deployment already has a holder. diff --git a/src/twinkle/server/gateway/proxy.py b/src/twinkle/server/gateway/proxy.py index 043e421e..3177a51c 100644 --- a/src/twinkle/server/gateway/proxy.py +++ b/src/twinkle/server/gateway/proxy.py @@ -10,11 +10,13 @@ import httpx from fastapi import Request, Response +from fastapi.responses import JSONResponse from typing import Any from twinkle.server.telemetry.tracing import inject_context from twinkle.utils.logger import get_logger from twinkle_client.http.headers import H_MULTIPLEX, H_MULTIPLEX_LEGACY, H_REQUEST_ID, H_REQUEST_ID_LEGACY +from twinkle_client.types.errors import ErrorCategory, ErrorPayload logger = get_logger() @@ -144,7 +146,19 @@ async def proxy_request( ) except Exception as e: logger.error('Proxy error: %s', str(e), exc_info=True) - return Response(content=f'Proxy Error: {str(e)}', status_code=502) + # The gateway could not reach the upstream deployment. Return the + # unified ``ErrorPayload`` (502/Server) instead of a plain-text body + # so every gateway failure has the same wire shape. Upstream error + # responses are passed through unchanged above, preserving their own + # ErrorPayload body. + request_id = request.headers.get(H_REQUEST_ID) or request.headers.get(H_REQUEST_ID_LEGACY) or '' + payload = ErrorPayload( + error=f'Proxy Error: {str(e)}', + category=ErrorCategory.Server, + error_code=502, + request_id=request_id, + ) + return JSONResponse(status_code=502, content=payload.model_dump(mode='json', exclude_none=True)) async def proxy_request_stream( self, diff --git a/src/twinkle/server/launcher/env_propagation.py b/src/twinkle/server/launcher/env_propagation.py index 4d4a7a47..2b333aba 100644 --- a/src/twinkle/server/launcher/env_propagation.py +++ b/src/twinkle/server/launcher/env_propagation.py @@ -33,9 +33,21 @@ def build_persistence_env_vars() -> dict[str, str]: return {k: os.environ[k] for k in PERSISTENCE_ENV_KEYS if k in os.environ} +def build_server_state_env_vars() -> dict[str, str]: + """Collect ServerState-policy env vars from ``os.environ`` for worker propagation. + + Read inside each worker by ``ServerStateArgs.from_env()`` (via + ``get_server_state``) so the configured quota / expiry / metrics interval is + applied everywhere, not only in the gateway that first built the state. + """ + from twinkle.server.config.application_spec import SERVER_STATE_ENV_KEYS + return {k: os.environ[k] for k in SERVER_STATE_ENV_KEYS if k in os.environ} + + def build_propagated_env_vars() -> dict[str, str]: """Aggregate all env vars that must reach Ray worker processes.""" merged: dict[str, str] = {} merged.update(build_telemetry_env_vars()) merged.update(build_persistence_env_vars()) + merged.update(build_server_state_env_vars()) return merged diff --git a/src/twinkle/server/launcher/server_launcher.py b/src/twinkle/server/launcher/server_launcher.py index 4c64c120..99ceaa74 100644 --- a/src/twinkle/server/launcher/server_launcher.py +++ b/src/twinkle/server/launcher/server_launcher.py @@ -223,6 +223,23 @@ def launch(self) -> None: os.environ[k] = v logger.info(f'Persistence backend configured: mode={persistence.mode}') + # Export the gateway ``server`` application's ServerState policy (quota / + # expiry / cleanup / metrics interval) to env vars for the same reason: + # so every worker's first ``get_server_state()`` applies the configured + # values instead of the hardcoded defaults. Without this the model worker + # that enforces ``per_token_model_limit`` runs on the default (30), + # silently ignoring the YAML value. + server_specs = [a for a in self.config.applications if a.import_path == 'server'] + if len(server_specs) > 1: + logger.warning(f'{len(server_specs)} "server" applications declared; using the first ' + 'for ServerState policy env propagation.') + if server_specs: + server_state_env = server_specs[0].args.server_config.to_env_vars() + for k, v in server_state_env.items(): + os.environ[k] = v + if server_state_env: + logger.info(f'ServerState policy exported to worker env: {server_state_env}') + model_alias_map = build_model_alias_map(self.config.applications) if model_alias_map: os.environ[MODEL_ID_ALIASES_ENV] = json.dumps(model_alias_map, ensure_ascii=False) diff --git a/src/twinkle/server/lifecycle/submit.py b/src/twinkle/server/lifecycle/submit.py index 005a2a3e..4bbc7017 100644 --- a/src/twinkle/server/lifecycle/submit.py +++ b/src/twinkle/server/lifecycle/submit.py @@ -23,8 +23,25 @@ # --------------------------------------------------------------------------- # -def to_backend_inputs(inputs: Any) -> Any: - """Seam A: convert raw dict/list inputs to InputFeature / Trajectory objects.""" +def to_backend_inputs(inputs: Any, *, single: bool = False) -> Any: + """Seam A: convert raw dict/list inputs to InputFeature / Trajectory objects. + + With ``single=False`` (default) a *batch* is returned: a list of parsed objects + for a list input, a one-element list for a single dict, and the value unchanged + otherwise. With ``single=True`` exactly one parsed object is returned (the + streaming path accepts only one input): a list must contain exactly one element + or a ``ValueError`` is raised, a dict is parsed to a single object, and anything + else is passed through. Element typing is unchanged: a dict with ``input_ids`` + becomes an ``InputFeature``, otherwise a ``Trajectory``. + """ + if single: + if isinstance(inputs, list): + if len(inputs) != 1: + raise ValueError('Streaming only supports a single input') + inputs = inputs[0] + if isinstance(inputs, dict): + return InputFeature(**inputs) if 'input_ids' in inputs else Trajectory(**inputs) + return inputs if isinstance(inputs, list) and inputs: first = inputs[0] if isinstance(first, dict) and 'input_ids' in first: diff --git a/src/twinkle/server/model/app.py b/src/twinkle/server/model/app.py index 77873b6d..42c4d799 100644 --- a/src/twinkle/server/model/app.py +++ b/src/twinkle/server/model/app.py @@ -20,7 +20,7 @@ from twinkle.server.state import ServerState, get_server_state from twinkle.server.utils import wrap_builder_with_device_group_env from twinkle.server.utils.backend_dispatch import BackendSelector -from twinkle.server.utils.lifecycle import AdapterManagerMixin +from twinkle.server.utils.session_resource import AdapterManagerMixin from twinkle.server.utils.task_queue import TaskQueueConfig, TaskQueueMixin from twinkle.server.utils.validation import get_token_from_request from twinkle.utils.logger import get_logger @@ -262,9 +262,9 @@ def assert_full_mode_available(self, adapter_name: str | None = None) -> None: """ if not self.is_full_mode: return - for rid, info in self._resource_records.items(): - if rid != adapter_name and not info.get('expiring'): - raise FullModeBusyError(rid) + holder = self.find_active_resource(exclude=adapter_name) + if holder is not None: + raise FullModeBusyError(holder) async def _on_adapter_expired(self, adapter_name: str) -> None: self.fail_pending_tasks_for_model(adapter_name, reason='Adapter expired') diff --git a/src/twinkle/server/model/twinkle_handlers.py b/src/twinkle/server/model/twinkle_handlers.py index ce0c724e..993ed214 100644 --- a/src/twinkle/server/model/twinkle_handlers.py +++ b/src/twinkle/server/model/twinkle_handlers.py @@ -172,7 +172,10 @@ async def _call(self, body, adapter_name, token): raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) ret = await self.call_backend( - self.model.forward, inputs=to_backend_inputs(raw_inputs), adapter_name=adapter_name, **kwargs) + self.model.forward, + inputs=to_backend_inputs(raw_inputs), + adapter_name=self.resolve_model_adapter_name(adapter_name), + **kwargs) return {'result': ret} return await run_submit( @@ -187,7 +190,11 @@ async def _call(self, body, adapter_name, token): raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) inputs = to_backend_inputs(raw_inputs) kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) - ret = await self.call_backend(self.model.forward_only, inputs=inputs, adapter_name=adapter_name, **kwargs) + ret = await self.call_backend( + self.model.forward_only, + inputs=inputs, + adapter_name=self.resolve_model_adapter_name(adapter_name), + **kwargs) if body.output_ref is not None: rows = select_output_rows(ret, batch_size=len(inputs), output_fields=body.output_fields) output_ref = await self.data_plane.append(body.output_ref, rows) @@ -211,7 +218,10 @@ async def _call(self, body, adapter_name, token): raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) ret = await self.call_backend( - self.model.forward_backward, inputs=to_backend_inputs(raw_inputs), adapter_name=adapter_name, **kwargs) + self.model.forward_backward, + inputs=to_backend_inputs(raw_inputs), + adapter_name=self.resolve_model_adapter_name(adapter_name), + **kwargs) return {'result': ret} return await run_submit( diff --git a/src/twinkle/server/processor/app.py b/src/twinkle/server/processor/app.py index 9f6b8896..b26bb323 100644 --- a/src/twinkle/server/processor/app.py +++ b/src/twinkle/server/processor/app.py @@ -22,7 +22,7 @@ from twinkle import DeviceGroup, DeviceMesh, get_logger from twinkle.server.deployment import LazyCleanupMixin, bind_deployment, build_deployment_app from twinkle.server.state import ServerState, get_server_state -from twinkle.server.utils.lifecycle import ProcessorManagerMixin +from twinkle.server.utils.session_resource import ProcessorManagerMixin from .twinkle_handlers import _register_processor_routes logger = get_logger() diff --git a/src/twinkle/server/sampler/backends/__init__.py b/src/twinkle/server/sampler/backends/__init__.py index 82c3255c..c2caf8e9 100644 --- a/src/twinkle/server/sampler/backends/__init__.py +++ b/src/twinkle/server/sampler/backends/__init__.py @@ -1,15 +1,11 @@ -STREAM_SENTINEL = '__STREAM_END__' +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Sampler backend implementations. +The cross-process streaming bridge lives in ``streaming`` and is re-exported here +so a sibling backend module can import it from ``.streaming`` without importing +this package root, while external consumers keep using +``from twinkle.server.sampler.backends import stream_to_queue``. +""" +from .streaming import STREAM_SENTINEL, stream_to_queue -def stream_to_queue(sampler, queue, inputs, sampling_params=None, adapter_name='', adapter_path=None): - """Push streaming deltas from *sampler* to a cross-process Ray queue. - - Works with any object that exposes a ``sample_stream`` iterator. - """ - try: - for delta, reason in sampler.sample_stream(inputs, sampling_params, adapter_name, adapter_path): - queue.put((delta, reason)) - except Exception as e: - queue.put(e) - finally: - queue.put(STREAM_SENTINEL) +__all__ = ['STREAM_SENTINEL', 'stream_to_queue'] diff --git a/src/twinkle/server/sampler/backends/mock_sampler.py b/src/twinkle/server/sampler/backends/mock_sampler.py index 2d5e5930..4d517903 100644 --- a/src/twinkle/server/sampler/backends/mock_sampler.py +++ b/src/twinkle/server/sampler/backends/mock_sampler.py @@ -168,7 +168,7 @@ def sample_stream( def sample_stream_to_queue(self, queue, inputs, sampling_params=None, adapter_name='', adapter_path=None): """Push streaming deltas to a cross-process Ray queue.""" - from . import stream_to_queue + from .streaming import stream_to_queue stream_to_queue(self, queue, inputs, sampling_params, adapter_name, adapter_path) @remote_function() diff --git a/src/twinkle/server/sampler/backends/streaming.py b/src/twinkle/server/sampler/backends/streaming.py new file mode 100644 index 00000000..dc61ff7e --- /dev/null +++ b/src/twinkle/server/sampler/backends/streaming.py @@ -0,0 +1,26 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Cross-process streaming bridge for sampler backends. + +Lives in its own module (rather than the ``backends`` package ``__init__``) so +that a backend implementation (e.g. ``mock_sampler``) can import it without +importing its own package root — which would be a package-initialisation-order +dependency. ``backends/__init__`` re-exports these names, so external consumers +(``from twinkle.server.sampler.backends import stream_to_queue``) are unchanged. +""" +from __future__ import annotations + +STREAM_SENTINEL = '__STREAM_END__' + + +def stream_to_queue(sampler, queue, inputs, sampling_params=None, adapter_name='', adapter_path=None): + """Push streaming deltas from *sampler* to a cross-process Ray queue. + + Works with any object that exposes a ``sample_stream`` iterator. + """ + try: + for delta, reason in sampler.sample_stream(inputs, sampling_params, adapter_name, adapter_path): + queue.put((delta, reason)) + except Exception as e: + queue.put(e) + finally: + queue.put(STREAM_SENTINEL) diff --git a/src/twinkle/server/sampler/tinker_handlers.py b/src/twinkle/server/sampler/tinker_handlers.py index 3f879001..927602b8 100644 --- a/src/twinkle/server/sampler/tinker_handlers.py +++ b/src/twinkle/server/sampler/tinker_handlers.py @@ -18,6 +18,7 @@ from twinkle.data_format import SamplingParams from twinkle.server.checkpoint import create_checkpoint_manager +from twinkle.server.sampler.weights import resolve_sampler_weights from twinkle.server.utils import get_template_for_model from twinkle.server.utils.task_queue.types import UserTaskError from twinkle.utils.logger import get_logger @@ -70,7 +71,10 @@ async def _do_sample(): # Set template for sampler based on model type template = get_template_for_model(self.model_id) await self.call_backend(self.sampler.set_template, template, model_id=self.model_id) - # Reset prefix cache for new weights + # Reset prefix cache unconditionally on every tinker request (by + # design): the tinker dialect does not signal whether weights + # changed, so it always invalidates. This differs from the twinkle + # endpoints, which reset only when an adapter_uri is supplied. await self.call_backend(self.sampler.reset_prefix_cache) # Get model_path from body or sampling session @@ -105,16 +109,10 @@ async def _do_sample(): logprobs=1, ) - # A resolved checkpoint is either a LoRA adapter dir (has - # adapter_config.json) or a full-parameter HF checkpoint. Full - # checkpoints are loaded into the sampler base model instead of - # being passed as a LoRA adapter. - lora_path = None - if adapter_uri: - if os.path.exists(os.path.join(adapter_uri, 'adapter_config.json')): - lora_path = adapter_uri - else: - await self.call_backend(self.sampler.load_full_weights_from_path, adapter_uri) + # LoRA adapter dir vs full-parameter checkpoint (shared helper); + # a full checkpoint is loaded into the base model and yields no + # LoRA path. + lora_path = await resolve_sampler_weights(self, adapter_uri) responses = await self.call_backend( self.sampler.sample, diff --git a/src/twinkle/server/sampler/twinkle_handlers.py b/src/twinkle/server/sampler/twinkle_handlers.py index ed58785d..0d85c2f5 100644 --- a/src/twinkle/server/sampler/twinkle_handlers.py +++ b/src/twinkle/server/sampler/twinkle_handlers.py @@ -21,9 +21,10 @@ import numpy as np import twinkle_client.types as types -from twinkle.data_format import InputFeature, SamplingParams, Trajectory +from twinkle.data_format import SamplingParams from twinkle.server.exceptions import RequestRejectedError -from twinkle.server.lifecycle.submit import resolve_twinkle_adapter_name +from twinkle.server.lifecycle.submit import resolve_twinkle_adapter_name, to_backend_inputs +from twinkle.server.sampler.weights import resolve_sampler_weights from twinkle.server.telemetry.correlation import MODEL_ID from twinkle.server.telemetry.tracing import traced_operation from twinkle.server.utils.task_errors import task_error_payload @@ -234,33 +235,15 @@ async def _task(): full_adapter_name = _get_twinkle_sampler_adapter_name(request, adapter_name) or '' if body.adapter_uri: - import os - from twinkle.server.checkpoint import create_checkpoint_manager checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') _, resolved_uri = checkpoint_manager.parse_adapter_uri(body.adapter_uri) - # Reset prefix cache only when new weights are loaded + # Reset prefix cache only when new weights are loaded. await self.call_backend(self.sampler.reset_prefix_cache) - # LoRA adapter dir (has adapter_config.json) vs full-parameter - # HF checkpoint. Full checkpoints replace the sampler base model. - if resolved_uri and os.path.exists(os.path.join(resolved_uri, 'adapter_config.json')): - adapter_path = resolved_uri - elif resolved_uri: - await self.call_backend(self.sampler.load_full_weights_from_path, resolved_uri) - - # Parse inputs - inputs = body.inputs - if isinstance(inputs, list) and inputs: - first = inputs[0] - if isinstance(first, dict) and 'input_ids' in first: - inputs = [InputFeature(**item) for item in inputs] - else: - inputs = [Trajectory(**item) for item in inputs] - elif isinstance(inputs, dict): - if 'input_ids' in inputs: - inputs = [InputFeature(**inputs)] - else: - inputs = [Trajectory(**inputs)] + adapter_path = await resolve_sampler_weights(self, resolved_uri) + + # Parse inputs (shared seam; batch form) + inputs = to_backend_inputs(body.inputs) # Build sampling params params = None @@ -304,14 +287,7 @@ async def sample_to_data_plane( _, adapter_path = checkpoint_manager.parse_adapter_uri(body.adapter_uri) inputs = (await self.data_plane.get(body.input_ref) if body.input_ref is not None else body.inputs) - if isinstance(inputs, list) and inputs: - first = inputs[0] - if isinstance(first, dict) and 'input_ids' in first: - inputs = [InputFeature(**item) for item in inputs] - else: - inputs = [Trajectory(**item) for item in inputs] - elif isinstance(inputs, dict): - inputs = [InputFeature(**inputs)] if 'input_ids' in inputs else [Trajectory(**inputs)] + inputs = to_backend_inputs(inputs) params_dict = dict(body.sampling_params or {}) params_dict['num_samples'] = body.num_samples @@ -431,29 +407,19 @@ async def sample_stream( full_adapter_name = _get_twinkle_sampler_adapter_name(request, adapter_name) or '' if body.adapter_uri: - import os - from twinkle.server.checkpoint import create_checkpoint_manager checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') _, resolved_uri = checkpoint_manager.parse_adapter_uri(body.adapter_uri) await self.call_backend(self.sampler.reset_prefix_cache) - if resolved_uri and os.path.exists(os.path.join(resolved_uri, 'adapter_config.json')): - adapter_path = resolved_uri - elif resolved_uri: - await self.call_backend(self.sampler.load_full_weights_from_path, resolved_uri) - - inputs = body.inputs - if isinstance(inputs, list): - if len(inputs) != 1: - raise HTTPException(status_code=400, detail='Streaming only supports a single input') - inputs = inputs[0] - if isinstance(inputs, dict): - if 'input_ids' in inputs: - inputs_parsed = InputFeature(**inputs) - else: - inputs_parsed = Trajectory(**inputs) - else: - inputs_parsed = inputs + adapter_path = await resolve_sampler_weights(self, resolved_uri) + + # Streaming accepts exactly one input; the shared seam enforces that and + # returns a single parsed object. Its ValueError maps to the same 400 this + # endpoint has always returned. + try: + inputs_parsed = to_backend_inputs(body.inputs, single=True) + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) params = None if body.sampling_params: diff --git a/src/twinkle/server/sampler/weights.py b/src/twinkle/server/sampler/weights.py new file mode 100644 index 00000000..2e5005df --- /dev/null +++ b/src/twinkle/server/sampler/weights.py @@ -0,0 +1,35 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Shared sampler weight-resolution rule (F004 / P004). + +The "a resolved checkpoint is either a LoRA adapter dir or a full-parameter HF +checkpoint" rule was re-derived from a filesystem probe in three sampler handlers +(``sample`` / ``sample_stream`` on the twinkle dialect and ``asample`` on tinker), +and the copies had begun to diverge. It lives here so the storage-layout decision +has a single owner. + +Prefix-cache invalidation is deliberately NOT handled here: each caller keeps its +own ``reset_prefix_cache`` policy (the tinker endpoint resets unconditionally on +every request; the twinkle endpoints reset only when an ``adapter_uri`` is +present), which is an observable behaviour difference this helper must not erase. +""" +from __future__ import annotations + +import os +from typing import Any + + +async def resolve_sampler_weights(service: Any, resolved_uri: str | None) -> str | None: + """Resolve a checkpoint path into a LoRA adapter path (or load full weights). + + Returns the LoRA ``adapter_path`` when ``resolved_uri`` is a directory holding + an ``adapter_config.json``. Otherwise the path is a full-parameter checkpoint: + it is loaded into the sampler base model via ``load_full_weights_from_path`` and + ``None`` is returned (no LoRA adapter to pass). ``None``/empty input returns + ``None`` unchanged. + """ + if not resolved_uri: + return None + if os.path.exists(os.path.join(resolved_uri, 'adapter_config.json')): + return resolved_uri + await service.call_backend(service.sampler.load_full_weights_from_path, resolved_uri) + return None diff --git a/src/twinkle/server/state/server_state.py b/src/twinkle/server/state/server_state.py index 2317558d..b675f976 100644 --- a/src/twinkle/server/state/server_state.py +++ b/src/twinkle/server/state/server_state.py @@ -617,14 +617,21 @@ async def get_cleanup_stats(self) -> dict[str, Any]: _PROCESS_STATE_CACHE: dict[str, ServerState] = {} +# ServerState policy defaults. Used when neither an explicit argument nor a +# launcher-propagated env var (``ServerStateArgs.from_env``) supplies a value. +_DEFAULT_EXPIRATION_TIMEOUT = 86400.0 # 24 hours in seconds +_DEFAULT_CLEANUP_INTERVAL = 3600.0 # 1 hour in seconds +_DEFAULT_PER_TOKEN_MODEL_LIMIT = 30 +_DEFAULT_METRICS_UPDATE_INTERVAL = 15.0 + def get_server_state(actor_name: str = 'twinkle_server_state', backend: StateBackend | None = None, persistence_config: PersistenceConfig | None = None, - expiration_timeout: float = 86400.0, - cleanup_interval: float = 3600.0, - per_token_model_limit: int = 30, - metrics_update_interval: float = 15.0) -> ServerState: + expiration_timeout: float | None = None, + cleanup_interval: float | None = None, + per_token_model_limit: int | None = None, + metrics_update_interval: float | None = None) -> ServerState: """Return a process-local :class:`ServerState` bound directly to the backend. Within one process the same ``actor_name`` returns the same cached instance @@ -655,6 +662,28 @@ def get_server_state(actor_name: str = 'twinkle_server_state', if cached is not None: return cached + # Resolve the ServerState policy: an explicit argument wins, else the + # launcher-propagated env (so a non-gateway worker honours the operator's + # YAML instead of the hardcoded default), else the module default. + from twinkle.server.config.application_spec import ServerStateArgs + env_policy = ServerStateArgs.from_env() + + def _resolve(explicit, env_value, default): + if explicit is not None: + return explicit + if env_value is not None: + return env_value + return default + + expiration_timeout = _resolve(expiration_timeout, getattr(env_policy, 'expiration_timeout', None), + _DEFAULT_EXPIRATION_TIMEOUT) + cleanup_interval = _resolve(cleanup_interval, getattr(env_policy, 'cleanup_interval', None), + _DEFAULT_CLEANUP_INTERVAL) + per_token_model_limit = _resolve(per_token_model_limit, getattr(env_policy, 'per_token_model_limit', None), + _DEFAULT_PER_TOKEN_MODEL_LIMIT) + metrics_update_interval = _resolve(metrics_update_interval, getattr(env_policy, 'metrics_update_interval', None), + _DEFAULT_METRICS_UPDATE_INTERVAL) + state = ServerState( backend=backend, persistence_config=persistence_config, @@ -664,6 +693,10 @@ def get_server_state(actor_name: str = 'twinkle_server_state', metrics_update_interval=metrics_update_interval, ) _PROCESS_STATE_CACHE[actor_name] = state + logger.info( + 'ServerState policy in effect: per_token_model_limit=%s expiration_timeout=%s ' + 'cleanup_interval=%s metrics_update_interval=%s (resolution: explicit>env>default)', per_token_model_limit, + expiration_timeout, cleanup_interval, metrics_update_interval) # Cleanup task is started by the deployment's FastAPI ``lifespan`` hook # via ``await state.start_cleanup_task()`` — that's the single async # entry point each worker has, so we don't need any sync-context diff --git a/src/twinkle/server/telemetry/middleware.py b/src/twinkle/server/telemetry/middleware.py index f6140da6..4745dd9d 100644 --- a/src/twinkle/server/telemetry/middleware.py +++ b/src/twinkle/server/telemetry/middleware.py @@ -24,7 +24,7 @@ from pydantic import BaseModel, ConfigDict from typing import Any -from twinkle.server.telemetry import MetricsRegistry +from twinkle.server.telemetry.metrics import MetricsRegistry from twinkle.utils.logger import get_logger logger = get_logger() diff --git a/src/twinkle/server/telemetry/worker_init.py b/src/twinkle/server/telemetry/worker_init.py index dcf53b90..c3aebb75 100644 --- a/src/twinkle/server/telemetry/worker_init.py +++ b/src/twinkle/server/telemetry/worker_init.py @@ -37,8 +37,9 @@ def ensure_telemetry_initialized() -> None: return try: - from twinkle.server.telemetry import TelemetryConfig, init_telemetry + from twinkle.server.config.telemetry import TelemetryConfig from twinkle.server.telemetry.metrics import MetricsRegistry + from twinkle.server.telemetry.provider import init_telemetry config = TelemetryConfig( enabled=True, @@ -82,7 +83,7 @@ def flush_telemetry_safely() -> None: so every error here is swallowed. """ try: - from twinkle.server.telemetry import shutdown_telemetry + from twinkle.server.telemetry.provider import shutdown_telemetry shutdown_telemetry() except Exception as e: # pragma: no cover - defensive logger.warning(f'Telemetry shutdown failed: {e}') diff --git a/src/twinkle/server/utils/__init__.py b/src/twinkle/server/utils/__init__.py index d6d484ab..bddeed29 100644 --- a/src/twinkle/server/utils/__init__.py +++ b/src/twinkle/server/utils/__init__.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from .device_utils import auto_fill_device_group_visible_devices, wrap_builder_with_device_group_env -from .lifecycle import AdapterManagerMixin, ProcessorManagerMixin, SessionResourceMixin +from .session_resource import AdapterManagerMixin, ProcessorManagerMixin, SessionResourceMixin from .task_queue import QueueState, RateLimiter, TaskQueueConfig, TaskQueueMixin, TaskStatus from .template_utils import get_template_for_model diff --git a/src/twinkle/server/utils/lifecycle/__init__.py b/src/twinkle/server/utils/lifecycle/__init__.py deleted file mode 100644 index ea574000..00000000 --- a/src/twinkle/server/utils/lifecycle/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) ModelScope Contributors. All rights reserved. -"""Lifecycle management utilities for session-bound resources.""" - -from .adapter import AdapterManagerMixin -from .base import SessionResourceMixin -from .processor import ProcessorManagerMixin - -__all__ = ['AdapterManagerMixin', 'ProcessorManagerMixin', 'SessionResourceMixin'] diff --git a/src/twinkle/server/utils/session_resource/__init__.py b/src/twinkle/server/utils/session_resource/__init__.py new file mode 100644 index 00000000..fdb7ebdc --- /dev/null +++ b/src/twinkle/server/utils/session_resource/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Session-bound resource lifecycle utilities (adapters / processors). + +Named ``session_resource`` (not ``lifecycle``) to avoid colliding with +``twinkle.server.lifecycle``, which is the *request* lifecycle (submit / retrieve +/ envelope). This package is the *resource* lifecycle: registration, heartbeat and +session-driven expiration of session-bound resources. +""" + +from .adapter import AdapterManagerMixin +from .base import SessionResourceMixin +from .processor import ProcessorManagerMixin + +__all__ = ['AdapterManagerMixin', 'ProcessorManagerMixin', 'SessionResourceMixin'] diff --git a/src/twinkle/server/utils/lifecycle/adapter.py b/src/twinkle/server/utils/session_resource/adapter.py similarity index 75% rename from src/twinkle/server/utils/lifecycle/adapter.py rename to src/twinkle/server/utils/session_resource/adapter.py index b23b84bb..5f51a722 100644 --- a/src/twinkle/server/utils/lifecycle/adapter.py +++ b/src/twinkle/server/utils/session_resource/adapter.py @@ -11,8 +11,6 @@ """ from __future__ import annotations -from typing import Any - from twinkle.utils.logger import get_logger from .base import SessionResourceMixin @@ -29,9 +27,9 @@ class AdapterManagerMixin(SessionResourceMixin): 1. Call _init_adapter_manager() in __init__ 2. Override _on_adapter_expired() to customize expiration handling - Attributes: - _adapter_timeout: Session inactivity timeout in seconds used to determine if a session is alive. - _adapter_max_lifetime: Maximum lifetime in seconds for any adapter, regardless of session liveness. + The inactivity timeout / max lifetime are stored on the base mixin as + ``_resource_timeout`` / ``_resource_max_lifetime`` (set via + ``_init_adapter_manager``). """ # Set resource type for logging @@ -57,23 +55,13 @@ def _init_adapter_manager( resource_max_lifetime=adapter_max_lifetime, ) - @property - def _adapter_timeout(self) -> float: - """Adapter timeout for backward compatibility.""" - return self._resource_timeout - - @property - def _adapter_max_lifetime(self) -> float | None: - """Adapter max lifetime for backward compatibility.""" - return self._resource_max_lifetime - - @property - def _adapter_records(self) -> dict[str, dict[str, Any]]: - """Adapter records for backward compatibility.""" - return self._resource_records - async def _on_resource_expired(self, resource_id: str) -> None: - """Internal hook called by base class. Delegates to _on_adapter_expired.""" + """Base-class expiry hook; forwards to the domain hook ``_on_adapter_expired``. + + ``_on_adapter_expired`` is the supported extension point: the adapter-domain + name is kept deliberately so subclass authors override a method named for + adapters rather than the generic base-class resource hook. + """ await self._on_adapter_expired(resource_id) async def _on_adapter_expired(self, adapter_name: str) -> None: @@ -103,7 +91,3 @@ def get_adapter_name(adapter_name: str) -> str: The adapter name to use """ return adapter_name - - def stop_adapter_countdown(self) -> None: - """Stop the background countdown task.""" - self.stop_resource_countdown() diff --git a/src/twinkle/server/utils/lifecycle/base.py b/src/twinkle/server/utils/session_resource/base.py similarity index 90% rename from src/twinkle/server/utils/lifecycle/base.py rename to src/twinkle/server/utils/session_resource/base.py index 394cd220..83c26189 100644 --- a/src/twinkle/server/utils/lifecycle/base.py +++ b/src/twinkle/server/utils/session_resource/base.py @@ -15,6 +15,7 @@ if TYPE_CHECKING: from twinkle.server.state import ServerState +from twinkle.server.exceptions import ResourceNotFoundError from twinkle.utils.logger import get_logger logger = get_logger() @@ -210,15 +211,33 @@ def clear_resource_state(self, resource_id: str) -> None: return info['state'] = {} + def find_active_resource(self, exclude: str | None = None) -> str | None: + """Return the id of an active (registered, not expiring) resource, if any. + + "Active" means present in the records and not marked ``expiring``. + ``exclude`` (a resource id) is skipped so a caller can ask "is any *other* + resource active?" — e.g. a tenant re-issuing against its own resource must + not count itself. Returns the first matching id, or ``None`` when none is + active. This is the owner-side query that lets callers avoid reaching into + the private ``_resource_records`` dict. + """ + for rid, info in self._resource_records.items(): + if rid != exclude and not info.get('expiring'): + return rid + return None + def assert_resource_exists(self, resource_id: str) -> None: """Validate that a resource exists and is not expiring. Raises: - AssertionError: If resource not found or expiring. + ResourceNotFoundError: 404/User — resource absent or expiring. Raised + (not ``assert``-ed) so the check is classified as a user-facing + 404 rather than collapsing to a 500, and so it survives + ``python -O`` (which strips ``assert`` statements). """ info = self._resource_records.get(resource_id) - assert resource_id and info is not None and not info.get('expiring'), \ - f'{self._resource_type} {resource_id} not found' + if not (resource_id and info is not None and not info.get('expiring')): + raise ResourceNotFoundError(f'{self._resource_type} {resource_id} not found') @abstractmethod async def _on_resource_expired(self, resource_id: str) -> None: diff --git a/src/twinkle/server/utils/lifecycle/processor.py b/src/twinkle/server/utils/session_resource/processor.py similarity index 81% rename from src/twinkle/server/utils/lifecycle/processor.py rename to src/twinkle/server/utils/session_resource/processor.py index 0a86309d..8eebc051 100644 --- a/src/twinkle/server/utils/lifecycle/processor.py +++ b/src/twinkle/server/utils/session_resource/processor.py @@ -25,9 +25,9 @@ class ProcessorManagerMixin(SessionResourceMixin): 1. Call _init_processor_manager() in __init__ 2. Override _on_processor_expired() to handle cleanup - Attributes: - _processor_timeout: Session inactivity timeout in seconds. - _per_token_processor_limit: Maximum active processors per user token. + The inactivity timeout is stored on the base mixin as ``_resource_timeout`` + (set via ``_init_processor_manager``); ``_per_token_processor_limit`` caps the + active processors per user token. """ # Set resource type for logging @@ -52,16 +52,6 @@ def _init_processor_manager( ) self._per_token_processor_limit = per_token_processor_limit - @property - def _processor_timeout(self) -> float: - """Processor timeout for backward compatibility.""" - return self._resource_timeout - - @property - def _processor_records(self) -> dict[str, dict[str, Any]]: - """Processor records for backward compatibility.""" - return self._resource_records - def _validate_registration(self, resource_id: str, token: str, session_id: str) -> None: """Validate before registering a processor. Checks per-token limit. @@ -91,7 +81,12 @@ def _create_resource_record(self, token: str, session_id: str) -> dict[str, Any] } async def _on_resource_expired(self, resource_id: str) -> None: - """Internal hook called by base class. Delegates to _on_processor_expired.""" + """Base-class expiry hook; forwards to the domain hook ``_on_processor_expired``. + + ``_on_processor_expired`` is the supported extension point: the + processor-domain name is kept deliberately so subclass authors override a + method named for processors rather than the generic base-class hook. + """ self._on_processor_expired(resource_id) def _on_processor_expired(self, processor_id: str) -> None: @@ -103,7 +98,3 @@ def _on_processor_expired(self, processor_id: str) -> None: NotImplementedError: If not overridden. """ raise NotImplementedError(f'_on_processor_expired must be implemented by {self.__class__.__name__}') - - def stop_processor_countdown(self) -> None: - """Stop the background countdown task.""" - self.stop_resource_countdown() diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/utils/task_queue/mixin.py index 49037c78..04e45445 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/utils/task_queue/mixin.py @@ -510,7 +510,13 @@ def _schedule() -> None: self._event_loop.call_soon_threadsafe(_schedule) def get_queue_stats(self) -> dict[str, Any]: - """Return current compute queue statistics.""" + """Return current compute queue statistics. + + Not exposed over HTTP yet: no endpoint reads these three ``*_stats`` + helpers. Kept as the intended data source for a future observability + endpoint; if that endpoint never lands, delete them instead of leaving + them as dead reflection. + """ return { 'queue_size': sum(q.qsize() for q in self._compute_worker.task_queues.values()), @@ -526,11 +532,11 @@ def get_queue_stats(self) -> dict[str, Any]: } def get_rate_limit_stats(self, token: str) -> dict[str, Any]: - """Return rate-limiting stats for a user token.""" + """Return rate-limiting stats for a user token. Not exposed over HTTP yet (see get_queue_stats).""" return self._rate_limiter.get_stats(token) def get_rate_limiter_memory_stats(self) -> dict[str, Any]: - """Return memory usage statistics from the rate limiter.""" + """Return memory usage statistics from the rate limiter. Not exposed over HTTP yet (see get_queue_stats).""" return self._rate_limiter.get_memory_stats() async def shutdown_task_queue(self) -> None: diff --git a/src/twinkle/server/utils/task_queue/worker.py b/src/twinkle/server/utils/task_queue/worker.py index b0e5fc14..8a473dcc 100644 --- a/src/twinkle/server/utils/task_queue/worker.py +++ b/src/twinkle/server/utils/task_queue/worker.py @@ -14,6 +14,7 @@ from collections import deque from typing import TYPE_CHECKING, Any, Callable, Deque +from twinkle.server.exceptions import TwinkleServerError from twinkle.server.telemetry.correlation import MODEL_ID, TOKEN_ID from twinkle.server.telemetry.tracing import traced_operation from twinkle.server.utils.task_errors import task_error_payload @@ -291,6 +292,26 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue f'{exec_time:.2f}s, type={task_type}, queue_key={queue_key}') # Gate held by a leaked timed-out call -> 503/Server. await self._store_task_failed(task, error, QueueState.ACTIVE.value, error_code=503) + except TwinkleServerError as exc: + # A typed server error carries its own status + category (e.g. + # ResourceNotFoundError = 404/User from a deferred + # assert_resource_exists). Honour them instead of collapsing every + # such failure to 500/Server. Only Server-category errors keep a + # traceback; a User rejection does not. + task_status = 'failed' + exec_time = time.monotonic() - exec_start + is_server = exc.category is ErrorCategory.Server + if is_server: + logger.error(f'[ComputeWorker] Task {task.request_id} FAILED after {exec_time:.2f}s, ' + f'type={task_type}:\n{traceback.format_exc(limit=3)}') + await self._store_task_failed( + task, + f'{type(exc).__name__}: {exc}', + QueueState.ACTIVE.value, + error_code=exc.error_code, + category=exc.category, + traceback_text=traceback.format_exc() if is_server else None, + ) except Exception as exc: task_status = 'failed' exec_time = time.monotonic() - exec_start diff --git a/tests/server/config/server_config_4b_e2e.yaml b/tests/server/config/server_config_4b_e2e.yaml index 6a9dfd31..f32eec8b 100644 --- a/tests/server/config/server_config_4b_e2e.yaml +++ b/tests/server/config/server_config_4b_e2e.yaml @@ -36,6 +36,7 @@ applications: backend: transformers model_id: "ms://Qwen/Qwen3.5-4B" max_length: 10240 + max_loras: 10 nproc_per_node: 2 device_group: name: model diff --git a/tests/server/config/server_config_4b_e2e_megatron.yaml b/tests/server/config/server_config_4b_e2e_megatron.yaml index a6f19350..4d5e3b60 100644 --- a/tests/server/config/server_config_4b_e2e_megatron.yaml +++ b/tests/server/config/server_config_4b_e2e_megatron.yaml @@ -36,6 +36,7 @@ applications: backend: megatron model_id: "ms://Qwen/Qwen3.5-4B" max_length: 10240 + max_loras: 10 nproc_per_node: 4 device_group: name: model diff --git a/tests/server/config/test_server_config.py b/tests/server/config/test_server_config.py index 433f85a5..9c50cb2a 100644 --- a/tests/server/config/test_server_config.py +++ b/tests/server/config/test_server_config.py @@ -246,8 +246,34 @@ def test_data_plane_application_uses_its_own_strict_args_schema() -> None: ApplicationSpec.model_validate({ 'name': 'data-plane', 'import_path': 'data_plane', - 'args': {'unknown': True}, + 'args': { + 'unknown': True + }, + }) + + +def test_processor_queue_config_is_rejected() -> None: + # A processor deployment has no task queue; queue_config was silently ignored + # before and now fails validation (F013 / P009) naming the offending field. + ApplicationSpec.model_validate({ + 'name': 'processor', + 'import_path': 'processor', + 'args': { + 'ncpu_proc_per_node': 1 + }, + }) + with pytest.raises(ValidationError) as exc: + ApplicationSpec.model_validate({ + 'name': 'processor', + 'import_path': 'processor', + 'args': { + 'ncpu_proc_per_node': 1, + 'queue_config': { + 'rps_limit': 4 + } + }, }) + assert 'queue_config' in str(exc.value) def test_cookbook_examples_load() -> None: diff --git a/tests/server/integration/test_sft_e2e.py b/tests/server/integration/test_sft_e2e.py index 40c794b1..61f0bd3d 100644 --- a/tests/server/integration/test_sft_e2e.py +++ b/tests/server/integration/test_sft_e2e.py @@ -5,6 +5,12 @@ - Twinkle client x (transformers | megatron) - Tinker client x (transformers | megatron) +Each test also verifies save-LoRA + resume-training succeeds: after the training +loop it saves a checkpoint (Twinkle ``model.save``; Tinker ``save_state``), resumes +(Twinkle ``resume_from_checkpoint``; Tinker +``create_training_client_from_state_with_optimizer``), and runs a few more steps +that must complete without timeout. + Backend selection via env var TWINKLE_TEST_BACKEND (default: transformers). ## How to run @@ -45,6 +51,7 @@ create_tinker_training_client, create_twinkle_sft_model, get_backend, + init_tinker_client_session, init_twinkle_client_session, log, wait_for_server, @@ -52,6 +59,7 @@ # ── Configuration ── SFT_TRAIN_STEPS = 20 # 20 steps ensures enough training for both backends +SFT_RESUME_STEPS = 3 # post-resume steps that must run without timeout # ═══════════════════════════════════════════════════════════════════════════ @@ -107,7 +115,32 @@ def test_sft_twinkle(): # Assertions — both backends should report real loss via calculate_metric assert len(losses) >= 4, f'Expected at least 4 logged losses, got {len(losses)}' assert_loss_decreases(losses, 'sft_twinkle') - log(f'test_sft_twinkle PASSED (backend={backend})') + + # ── Save LoRA + resume training (must succeed) ── + save_resp = model.save( + name='sft-twinkle-resume', + save_optimizer=True, + consumed_train_samples=dataloader.get_state()['consumed_train_samples'], + ) + ckpt = save_resp.twinkle_path + assert ckpt, 'save() did not return a twinkle_path' + log(f'saved LoRA checkpoint: {ckpt}') + + progress = model.resume_from_checkpoint(ckpt) + log(f'resumed from checkpoint: {progress}') + + resume_loader = DataLoader(dataset=create_sft_dataset(), batch_size=4) + resumed = 0 + for step, batch in enumerate(resume_loader): + if step >= SFT_RESUME_STEPS: + break + t0 = time.time() + model.forward_backward(inputs=batch) + model.clip_grad_and_step() + assert_no_timeout(time.time() - t0, f'sft_twinkle resume step {step}') + resumed += 1 + assert resumed == SFT_RESUME_STEPS, f'expected {SFT_RESUME_STEPS} post-resume steps, ran {resumed}' + log(f'test_sft_twinkle PASSED (backend={backend}) [+save LoRA +resume]') # ═══════════════════════════════════════════════════════════════════════════ @@ -171,7 +204,31 @@ def test_sft_tinker(): # Assertions assert len(losses) >= 4, f'Expected at least 4 logged losses, got {len(losses)}' assert_loss_decreases(losses, 'sft_tinker') - log(f'test_sft_tinker PASSED (backend={backend})') + + # ── Save state + resume training (must succeed) ── + save_result = training_client.save_state('sft-tinker-resume').result() + state_path = save_result.path + assert state_path, 'save_state() did not return a path' + log(f'saved tinker state: {state_path}') + + # Resume restores both weights and optimizer state into a fresh client. + service_client = init_tinker_client_session() + resumed_client = service_client.create_training_client_from_state_with_optimizer(path=state_path) + log('resumed tinker training client from saved state') + + resume_loader = DataLoader(dataset=create_sft_dataset(), batch_size=4) + resumed = 0 + for step, batch in enumerate(resume_loader): + if step >= SFT_RESUME_STEPS: + break + input_datums = [input_feature_to_datum(input_feature) for input_feature in batch] + t0 = time.time() + resumed_client.forward_backward(input_datums, 'cross_entropy').result() + resumed_client.optim_step(types.AdamParams(learning_rate=1e-4)).result() + assert_no_timeout(time.time() - t0, f'sft_tinker resume step {step}') + resumed += 1 + assert resumed == SFT_RESUME_STEPS, f'expected {SFT_RESUME_STEPS} post-resume steps, ran {resumed}' + log(f'test_sft_tinker PASSED (backend={backend}) [+save state +resume]') # ── Direct execution ── diff --git a/tests/server/lifecycle/test_to_backend_inputs.py b/tests/server/lifecycle/test_to_backend_inputs.py new file mode 100644 index 00000000..4be37037 --- /dev/null +++ b/tests/server/lifecycle/test_to_backend_inputs.py @@ -0,0 +1,48 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Characterization tests for the shared ``to_backend_inputs`` seam (F003 / P003). + +Pins the input-shape rules that the sampler handlers used to re-implement inline, +so the three call sites (``sample`` / ``sample_to_data_plane`` batch form, and +``sample_stream`` single form) now share one definition. +""" +from __future__ import annotations + +import pytest + +from twinkle.data_format import InputFeature, Trajectory +from twinkle.server.lifecycle.submit import to_backend_inputs + +_IF = {'input_ids': [1, 2, 3]} +_TRAJ = {'messages': [{'role': 'user', 'content': 'hi'}]} + + +def test_batch_list_of_input_features(): + # Each element is parsed with InputFeature (dict with input_ids). + assert to_backend_inputs([_IF, _IF]) == [InputFeature(**_IF), InputFeature(**_IF)] + + +def test_batch_list_of_trajectories(): + # A dict without input_ids is parsed as a Trajectory. + assert to_backend_inputs([_TRAJ]) == [Trajectory(**_TRAJ)] + + +def test_batch_single_dict_becomes_one_element_list(): + assert to_backend_inputs(_IF) == [InputFeature(**_IF)] + assert to_backend_inputs(_TRAJ) == [Trajectory(**_TRAJ)] + + +def test_batch_passthrough_for_non_list_non_dict(): + sentinel = object() + assert to_backend_inputs(sentinel) is sentinel + + +def test_single_returns_one_object_not_a_list(): + out = to_backend_inputs([_IF], single=True) + assert not isinstance(out, list) + assert out == InputFeature(**_IF) + assert to_backend_inputs(_TRAJ, single=True) == Trajectory(**_TRAJ) + + +def test_single_rejects_multi_element_list(): + with pytest.raises(ValueError, match='single input'): + to_backend_inputs([_IF, _IF], single=True) diff --git a/tests/server/sampler/test_resolve_sampler_weights.py b/tests/server/sampler/test_resolve_sampler_weights.py new file mode 100644 index 00000000..1d98c76c --- /dev/null +++ b/tests/server/sampler/test_resolve_sampler_weights.py @@ -0,0 +1,53 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Tests for the shared sampler weight-resolution helper (F004 / P004).""" +from __future__ import annotations + +import pytest + +from twinkle.server.sampler.weights import resolve_sampler_weights + + +class _FakeSampler: + + def load_full_weights_from_path(self, path): + return None + + +class _FakeService: + """Minimal stand-in exposing the two attributes the helper touches.""" + + def __init__(self): + self.sampler = _FakeSampler() + self.full_weight_loads: list[str] = [] + + async def call_backend(self, fn, *args, **kwargs): + # The helper only routes load_full_weights_from_path through call_backend. + if getattr(fn, '__name__', None) == 'load_full_weights_from_path': + self.full_weight_loads.append(args[0]) + return fn(*args, **kwargs) + + +@pytest.mark.asyncio +async def test_lora_dir_returns_adapter_path_and_loads_no_full_weights(tmp_path): + (tmp_path / 'adapter_config.json').write_text('{}') + svc = _FakeService() + result = await resolve_sampler_weights(svc, str(tmp_path)) + assert result == str(tmp_path) + assert svc.full_weight_loads == [] + + +@pytest.mark.asyncio +async def test_full_checkpoint_loads_weights_and_returns_none(tmp_path): + # A directory without adapter_config.json is a full-parameter checkpoint. + svc = _FakeService() + result = await resolve_sampler_weights(svc, str(tmp_path)) + assert result is None + assert svc.full_weight_loads == [str(tmp_path)] + + +@pytest.mark.asyncio +async def test_empty_uri_is_a_noop(): + svc = _FakeService() + assert await resolve_sampler_weights(svc, None) is None + assert await resolve_sampler_weights(svc, '') is None + assert svc.full_weight_loads == [] diff --git a/tests/server/start_e2e_server.py b/tests/server/start_e2e_server.py index 0e8939ed..922283de 100644 --- a/tests/server/start_e2e_server.py +++ b/tests/server/start_e2e_server.py @@ -22,8 +22,8 @@ import requests # ── Paths ── -RAY = "/mnt/nas2/anaconda3/envs/tinker_myl/bin/ray" -PYTHON = "/mnt/nas2/anaconda3/envs/tinker_myl/bin/python" +RAY = "/mnt/nas2/anaconda3/envs/twinkle_ppu_vllm/bin/ray" +PYTHON = "/mnt/nas2/anaconda3/envs/twinkle_ppu_vllm/bin/python" WORKDIR = "/mnt/nas2/yunlin.myl/twinkle" DEFAULT_CONFIG = "tests/server/config/server_config_4b_e2e.yaml" RAY_TEMP_DIR = "/mnt/nas2/yunlin.myl/ray_logs" @@ -136,14 +136,14 @@ def restart_ray(): run(f"{RAY} stop --force", check=False) time.sleep(2) - # Head node: GPU 0,1,2,3 (4 GPUs for model PP=2 x DP=2) + # Head node: 4 GPUs for model PP=2 x DP=2 (skip busy card 2). run(f"{RAY} start --head --port=6379 --num-gpus=4 " f"--disable-usage-stats --temp-dir={RAY_TEMP_DIR}", - env={"CUDA_VISIBLE_DEVICES": "0,1,2,3"}) + env={"CUDA_VISIBLE_DEVICES": "0,1,3,4"}) - # Worker: GPU 4 (1 GPU for sampler) + # Worker: 1 GPU for sampler. run(f"{RAY} start --address=127.0.0.1:6379 --num-gpus=1", - env={"CUDA_VISIBLE_DEVICES": "4"}) + env={"CUDA_VISIBLE_DEVICES": "5"}) # CPU-only worker (processor + server) run(f"{RAY} start --address=127.0.0.1:6379 --num-gpus=0", diff --git a/tests/server/static/backend_call_exemptions.py b/tests/server/static/backend_call_exemptions.py index 4dcd3bf0..f3705fa6 100644 --- a/tests/server/static/backend_call_exemptions.py +++ b/tests/server/static/backend_call_exemptions.py @@ -10,14 +10,23 @@ relative to ``src/twinkle/server`` and ``function_name`` is the innermost enclosing function of the exempted call. -The only allowed exemption is the ray ``Queue.get`` inside ``sample_stream``'s -``_stream_generator``: it bridges the sampler actor's process boundary and is bounded -by the dedicated double-timeout of R4#10-11 (T5.5), not by ``call_backend``. No -``remote_function`` call is exempt. +The allowed exemptions are: + +- the ray ``Queue.get`` inside ``sample_stream``'s ``_stream_queue``: it bridges the + sampler actor's process boundary and is bounded by the dedicated double-timeout of + R4#10-11 (T5.5), not by ``call_backend``; +- the ``.sample_stream_to_queue.remote(...)`` call inside ``sample_stream`` + itself: streaming generation must keep producing while the HTTP response streams, + so it cannot use ``call_backend`` as-is and carries its own double timeout. This is + a ``remote_function`` bypass that the guard now *detects* (via backend-derived + local tracking) and that is *explicitly* accepted here — replacing the previous + "No remote_function call is exempt" claim, which was true only because the guard + could not see this shape. """ from __future__ import annotations # (module_relpath under src/twinkle/server, innermost enclosing function name) BACKEND_CALL_EXEMPTIONS: frozenset[tuple[str, str]] = frozenset({ ('sampler/twinkle_handlers.py', '_stream_queue'), + ('sampler/twinkle_handlers.py', 'sample_stream'), }) diff --git a/tests/server/static/test_adapter_name_mapping.py b/tests/server/static/test_adapter_name_mapping.py new file mode 100644 index 00000000..6856f099 --- /dev/null +++ b/tests/server/static/test_adapter_name_mapping.py @@ -0,0 +1,71 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Static check: model backend calls map the tenant adapter name (F002 / P002). + +``run_submit`` hands each ``backend_call`` a *tenant-scoped* adapter name +(``owner_id-``). Before that reaches a backend model method it must be +translated by ``ModelManagement.resolve_model_adapter_name`` — which returns +``''`` in full-parameter mode (the empty-string default optimizer group) and the +name unchanged in LoRA mode. The inline forward/backward/step endpoints did this; +the ``*_from_data_plane`` endpoints originally passed the raw tenant name, so a +full-mode data-plane request drove the wrong optimizer group. + +This guard fails if any ``self.call_backend(...)`` in ``model/twinkle_handlers.py`` +passes ``adapter_name`` as the bare local (i.e. unmapped). Passing the raw name +*positionally* (as ``add_adapter_to_model`` does when creating a tenant adapter) +is intentionally not a keyword and is therefore not flagged. +""" +from __future__ import annotations + +import ast +import pathlib + +import twinkle + +_HANDLERS = pathlib.Path(twinkle.__file__).resolve().parent / 'server' / 'model' / 'twinkle_handlers.py' + + +def _is_call_backend(func: ast.AST) -> bool: + """True for ``self.call_backend`` (Attribute ``call_backend`` on Name ``self``).""" + return (isinstance(func, ast.Attribute) and func.attr == 'call_backend' and isinstance(func.value, ast.Name) + and func.value.id == 'self') + + +def _is_mapped(value: ast.AST) -> bool: + """True when the ``adapter_name`` value is wrapped by ``self.resolve_model_adapter_name(...)``.""" + return (isinstance(value, ast.Call) and isinstance(value.func, ast.Attribute) + and value.func.attr == 'resolve_model_adapter_name') + + +def _unmapped_adapter_name_calls(tree: ast.AST) -> list[tuple[int, str]]: + offenders: list[tuple[int, str]] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or not _is_call_backend(node.func): + continue + for kw in node.keywords: + if kw.arg != 'adapter_name': + continue + # Bare local ``adapter_name`` is the tenant-scoped name and must be mapped. + if isinstance(kw.value, ast.Name) and kw.value.id == 'adapter_name' and not _is_mapped(kw.value): + offenders.append((node.lineno, ast.unparse(kw.value))) + return offenders + + +def test_model_backend_calls_map_adapter_name(): + tree = ast.parse(_HANDLERS.read_text(), filename=str(_HANDLERS)) + offenders = _unmapped_adapter_name_calls(tree) + assert not offenders, ('These self.call_backend(...) sites pass the raw tenant adapter_name instead of ' + f'self.resolve_model_adapter_name(adapter_name): {offenders}') + + +def test_checker_detects_unmapped_adapter_name(): + source = ('async def route(self, body, adapter_name, token):\n' + ' await self.call_backend(self.model.forward, inputs=[], adapter_name=adapter_name)\n') + assert _unmapped_adapter_name_calls(ast.parse(source)) == [(2, 'adapter_name')] + + +def test_checker_allows_mapped_and_positional(): + source = ( + 'async def route(self, body, adapter_name, token):\n' + ' await self.call_backend(self.model.forward, adapter_name=self.resolve_model_adapter_name(adapter_name))\n' + ' await self.call_backend(self.model.add_adapter_to_model, adapter_name, config)\n') + assert _unmapped_adapter_name_calls(ast.parse(source)) == [] diff --git a/tests/server/static/test_no_direct_backend_call.py b/tests/server/static/test_no_direct_backend_call.py index 5b218e91..44449038 100644 --- a/tests/server/static/test_no_direct_backend_call.py +++ b/tests/server/static/test_no_direct_backend_call.py @@ -43,12 +43,32 @@ def _getattr_backend_method(node: ast.AST) -> str | None: return None +def _is_backend_derived(node: ast.AST, backend_names: set[str]) -> bool: + """True if *node* is (transitively) the backend or a value bound from it. + + Matches ``self.model`` / ``self.sampler`` and any attribute/subscript chain + rooted at them, plus locals recorded in ``backend_names`` (e.g. from + ``actors = self.sampler._actors`` then ``actor = actors[0]``). This catches the + "bind the private actor list to a local, then call ``.remote()``" bypass that + plain attribute-name matching cannot see. + """ + while isinstance(node, (ast.Attribute, ast.Subscript)): + if isinstance(node, ast.Attribute): + if isinstance(node.value, ast.Name) and node.value.id == 'self' and node.attr in ('model', 'sampler'): + return True + node = node.value + else: + node = node.value + return isinstance(node, ast.Name) and node.id in backend_names + + class _Collector(ast.NodeVisitor): def __init__(self, relpath: str) -> None: self.relpath = relpath self.func_stack: list[str] = [] self.backend_aliases: set[str] = set() + self.backend_derived: set[str] = set() self.offenders: list[tuple[str, str, int, str]] = [] def _visit_func(self, node: ast.AST) -> None: @@ -62,6 +82,10 @@ def _visit_func(self, node: ast.AST) -> None: def visit_Assign(self, node: ast.Assign) -> None: if _getattr_backend_method(node.value) is not None: self.backend_aliases.update(target.id for target in node.targets if isinstance(target, ast.Name)) + # Track locals bound (transitively) to the private backend actor list, so a + # later ``.remote()`` on them is still counted as a backend call. + if _is_backend_derived(node.value, self.backend_derived): + self.backend_derived.update(target.id for target in node.targets if isinstance(target, ast.Name)) self.generic_visit(node) def visit_Call(self, node: ast.Call) -> None: @@ -75,6 +99,12 @@ def visit_Call(self, node: ast.Call) -> None: if escaped_owner is not None: owner = escaped_owner label = f'{ast.unparse(node.func)}({ast.unparse(node.args[0])})' + # ``..remote(...)`` where traces back to the private + # backend actor list bypasses call_backend just like a direct call. + if (owner is None and isinstance(node.func, ast.Attribute) and node.func.attr == 'remote' + and _is_backend_derived(node.func.value, self.backend_derived)): + owner = 'remote' + label = ast.unparse(node.func) if owner is not None: enclosing = self.func_stack[-1] if self.func_stack else '' if (self.relpath, enclosing) not in BACKEND_CALL_EXEMPTIONS: @@ -90,9 +120,8 @@ def test_no_direct_backend_call_in_server(): collector.visit(ast.parse(path.read_text(), filename=str(path))) offenders.extend(collector.offenders) - assert not offenders, ( - 'Direct backend calls must go through call_backend (or be listed in ' - f'backend_call_exemptions): {offenders}') + assert not offenders, ('Direct backend calls must go through call_backend (or be listed in ' + f'backend_call_exemptions): {offenders}') def test_exemptions_are_read_from_shared_file(): @@ -120,3 +149,31 @@ async def route(self): collector = _Collector('example.py') collector.visit(ast.parse(source)) assert collector.offenders == [] + + +def test_checker_detects_aliased_remote_backend_call(): + # The sample_stream bypass shape: bind the private actor list to a local, then + # call .remote() on an element. The guard must see this through the aliasing. + source = """ +async def sample_stream(self): + actors = self.sampler._actors + actor = actors[0] + actor.sample_stream_to_queue.remote(q) +""" + collector = _Collector('example.py') + collector.visit(ast.parse(source)) + assert len(collector.offenders) == 1 + assert collector.offenders[0][1] == 'sample_stream' + + +def test_checker_ignores_remote_on_unrelated_object(): + # A .remote() on a handle not derived from self.model/self.sampler is not a + # backend-boundary bypass and must not be flagged. + source = """ +async def route(self): + handle = get_some_actor() + handle.do.remote(1) +""" + collector = _Collector('example.py') + collector.visit(ast.parse(source)) + assert collector.offenders == [] diff --git a/tests/server/static/test_no_package_root_imports.py b/tests/server/static/test_no_package_root_imports.py new file mode 100644 index 00000000..b14c0f78 --- /dev/null +++ b/tests/server/static/test_no_package_root_imports.py @@ -0,0 +1,61 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Architecture check (F007 / P006): no module imports its own package root. + +A module under ``twinkle.server`` must not import one of its own *ancestor* +sub-packages (e.g. ``checkpoint/tinker.py`` importing ``twinkle.server.checkpoint``). +Such an import is a package-initialisation-order dependency: it re-enters the +ancestor's ``__init__`` while that ``__init__`` is still importing the child, +which is the module-level cycle this check forbids. Importing a *sibling* module +directly (``from .paths import ...``) or the top-level ``twinkle`` public API is +fine and excluded. + +``import-linter``'s ``forbidden`` contract cannot express this (it treats the +source module as part of the forbidden package and reports the contract as kept), +so the check is written directly against grimp's static graph. grimp parses files +without importing the target package, so this stays safe to run in CI. +""" +from __future__ import annotations + +import pathlib +import pytest + +grimp = pytest.importorskip('grimp', reason='grimp is required for the package-root import architecture check') + +import twinkle # noqa: E402 + +_PACKAGE = 'twinkle.server' +_SRC = str(pathlib.Path(twinkle.__file__).resolve().parent.parent) + + +def _package_root_imports() -> list[dict]: + """Return every import where a module imports one of its own ancestor sub-packages.""" + import sys + if _SRC not in sys.path: + sys.path.insert(0, _SRC) + graph = grimp.build_graph('twinkle', include_external_packages=False) + offenders: list[dict] = [] + for module in sorted(graph.modules): + if not module.startswith(_PACKAGE + '.'): + continue + parts = module.split('.') + ancestors = {'.'.join(parts[:i]) for i in range(1, len(parts))} + ancestors = {a for a in ancestors if a.startswith(_PACKAGE + '.')} + for imported in graph.find_modules_directly_imported_by(module): + if imported in ancestors: + for detail in graph.get_import_details(importer=module, imported=imported): + offenders.append({ + 'importer': module, + 'imported': imported, + 'line_number': detail.get('line_number'), + 'line_contents': (detail.get('line_contents') or '').strip(), + }) + return offenders + + +def test_no_module_imports_its_own_package_root(): + offenders = _package_root_imports() + assert not offenders, ( + 'These modules import one of their own ancestor sub-packages (a package-init cycle); ' + 'import the sibling module directly instead:\n' + + '\n'.join(f" {o['importer']} -> {o['imported']} (L{o['line_number']}): {o['line_contents']}" + for o in offenders)) diff --git a/tests/server/test_deployment_exception_boundary.py b/tests/server/test_deployment_exception_boundary.py index 4f61b42e..1e9c9fe0 100644 --- a/tests/server/test_deployment_exception_boundary.py +++ b/tests/server/test_deployment_exception_boundary.py @@ -34,8 +34,14 @@ async def boom(): response = client.get('/boom', headers={'x-request-id': 'boundary-test'}) assert response.status_code == 500 assert response.headers['X-Twinkle-Replica-Id'] == 'replica-test' - assert 'Traceback' in response.json()['detail'] - assert 'RuntimeError: boom with replica header' in response.json()['detail'] + # Unhandled exceptions now return the unified ErrorPayload (Server category + # keeps the traceback) instead of the legacy {'detail': } shape. + body = response.json() + assert body['category'] == 'server' + assert body['error_code'] == 500 + assert body['error'] == 'boom with replica header' + assert 'Traceback' in body['traceback'] + assert 'RuntimeError: boom with replica header' in body['traceback'] response = client.get('/healthz') assert response.status_code == 200 diff --git a/tests/server/utils/test_task_queue_mixin.py b/tests/server/utils/test_task_queue_mixin.py index 74447e5e..98aaf5e7 100644 --- a/tests/server/utils/test_task_queue_mixin.py +++ b/tests/server/utils/test_task_queue_mixin.py @@ -1,5 +1,4 @@ import asyncio - import pytest from twinkle.server.utils.task_queue.config import TaskQueueConfig @@ -148,11 +147,7 @@ async def work(): try: await queue.schedule_task(work, model_id='model1', token='token1') for _ in range(100): - completed = [ - kwargs - for args, kwargs in queue.state.records - if args[1] == 'completed' - ] + completed = [kwargs for args, kwargs in queue.state.records if args[1] == 'completed'] if completed: break await asyncio.sleep(0) @@ -212,6 +207,34 @@ async def work(): assert 'traceback' not in failed[-1]['result'] +@pytest.mark.asyncio +async def test_typed_server_error_keeps_its_status_and_category(): + """A TwinkleServerError (e.g. ResourceNotFoundError) must keep its own 404/user + classification instead of collapsing to a generic 500/server.""" + from twinkle.server.exceptions import ResourceNotFoundError + + queue = _DummyQueue() + queue.enable_compute_worker() + + async def work(): + raise ResourceNotFoundError('adapter foo not found') + + try: + await queue.schedule_task(work, model_id='model1', token='token1') + for _ in range(100): + failed = [kwargs for args, kwargs in queue.state.records if args[1] == 'failed'] + if failed: + break + await asyncio.sleep(0) + finally: + await queue._compute_worker.stop() + + assert failed[-1]['result']['error_code'] == 404 + assert failed[-1]['result']['category'] == 'user' + # A user rejection carries no traceback. + assert 'traceback' not in failed[-1]['result'] + + @pytest.mark.asyncio async def test_submit_and_peek_preflight_rejection_raises_without_writing_or_queuing(): from twinkle.server.exceptions import BatchSizeError From b928a171eb3bc45ceb2ad30629e069b7b68d40d5 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Fri, 18 Sep 2026 03:52:48 +0000 Subject: [PATCH 18/32] =?UTF-8?q?feat(server):=20request=20schema=20?= =?UTF-8?q?=E2=80=94=20strict=20wire=20models,=20preflight,=20and=20naming?= =?UTF-8?q?=20disambiguation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move HTTP-boundary-decidable request problems out of the async training path: - Single shared request models (twinkle_client.types) with field roles (control / backend_kwarg / passthrough); client builds via build_request + model_dump_json instead of hand-assembling json_data. - StrictRequest on all twinkle-native routes; unified RequestValidationError -> ErrorPayload (422/501) registered on the shared deployment app builder. - Wire schema for inline `inputs` validated at HTTP ingress (strict int leaves, homogeneous batch, extension-field preservation); single shared `is_encoded` predicate in twinkle.data_format.encoding replaces three copies. - run_submit preflight (assert_request_supported) rejects backend-incompatible fields and unavailable endpoints before seq claim / enqueue -> zero DP ranks. - Passthrough keys forwarded unjudged (no spelling heuristic). Naming disambiguation (client/server contract fix): - Prefix sampler-domain models (SamplerAddAdapterRequest / SetTemplate* / CreateResponse) so they no longer collide with model.py; the sampler handler now binds sampler_types explicitly. Fixes add_adapter_to_sampler validating against model.py's `config: Optional[str]` and rejecting the dict the client sends. - Remove dead model.AddAdapterResponse and server.WeightsInfoResponse. - training.py response envelopes inherit ResponseModel. - Regenerate contract route inventory; add regression tests pinning the sampler binding and the dict-config acceptance. --- src/twinkle/data_format/__init__.py | 1 + src/twinkle/data_format/encoding.py | 32 ++ src/twinkle/model/megatron/megatron.py | 4 +- .../model/transformers/transformers.py | 4 +- src/twinkle/sampler/base.py | 7 +- src/twinkle/server/deployment.py | 22 +- src/twinkle/server/lifecycle/submit.py | 128 +++-- src/twinkle/server/model/twinkle_handlers.py | 49 +- .../server/processor/twinkle_handlers.py | 4 +- .../server/sampler/twinkle_handlers.py | 37 +- src/twinkle/server/validation/__init__.py | 32 ++ .../server/validation/backend_compat.py | 125 +++++ src/twinkle/server/validation/errors.py | 77 +++ src/twinkle_client/_request_builder.py | 110 ++++ src/twinkle_client/common/__init__.py | 7 + src/twinkle_client/common/component_rpc.py | 53 ++ src/twinkle_client/dataloader/dataloader.py | 108 +--- src/twinkle_client/dataset/base.py | 228 ++------ .../dataset/iterable_dataset.py | 84 +-- .../dataset/iterable_packing_dataset.py | 86 +-- src/twinkle_client/dataset/lazy_dataset.py | 158 ++---- src/twinkle_client/dataset/packing_dataset.py | 58 +- src/twinkle_client/exceptions.py | 15 + src/twinkle_client/http/__init__.py | 3 +- src/twinkle_client/http/http_utils.py | 19 + .../model/multi_lora_transformers.py | 522 ++++++++++-------- src/twinkle_client/processor/base.py | 46 +- src/twinkle_client/sampler/vllm_sampler.py | 110 ++-- src/twinkle_client/types/__init__.py | 40 +- src/twinkle_client/types/base.py | 133 ++++- src/twinkle_client/types/component.py | 56 +- src/twinkle_client/types/data.py | 216 ++++++++ src/twinkle_client/types/model.py | 414 ++++++++------ src/twinkle_client/types/processor.py | 47 +- src/twinkle_client/types/sampler.py | 67 ++- src/twinkle_client/types/server.py | 29 +- src/twinkle_client/types/session.py | 12 +- src/twinkle_client/types/training.py | 8 +- src/twinkle_client/utils/__init__.py | 7 + tests/server/contract/client_api_routes.json | 18 +- .../server/lifecycle/test_run_submit_dedup.py | 15 +- tests/server/validation/__init__.py | 0 tests/server/validation/test_preflight.py | 243 ++++++++ tests/server/validation/test_request_wire.py | 234 ++++++++ tests/server/validation/test_wire_schema.py | 209 +++++++ tests/twinkle_client/test_async_components.py | 117 ++-- tests/twinkle_client/test_request_builder.py | 156 ++++++ 47 files changed, 2840 insertions(+), 1310 deletions(-) create mode 100644 src/twinkle/data_format/encoding.py create mode 100644 src/twinkle/server/validation/__init__.py create mode 100644 src/twinkle/server/validation/backend_compat.py create mode 100644 src/twinkle/server/validation/errors.py create mode 100644 src/twinkle_client/_request_builder.py create mode 100644 src/twinkle_client/common/__init__.py create mode 100644 src/twinkle_client/common/component_rpc.py create mode 100644 src/twinkle_client/types/data.py create mode 100644 src/twinkle_client/utils/__init__.py create mode 100644 tests/server/validation/__init__.py create mode 100644 tests/server/validation/test_preflight.py create mode 100644 tests/server/validation/test_request_wire.py create mode 100644 tests/server/validation/test_wire_schema.py create mode 100644 tests/twinkle_client/test_request_builder.py diff --git a/src/twinkle/data_format/__init__.py b/src/twinkle/data_format/__init__.py index 5db25a2b..93946382 100644 --- a/src/twinkle/data_format/__init__.py +++ b/src/twinkle/data_format/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. +from .encoding import ENCODED_INPUT_KEYS, is_encoded from .input_feature import InputFeature from .message import Message, Tool, ToolCall from .output import LossOutput, ModelOutput diff --git a/src/twinkle/data_format/encoding.py b/src/twinkle/data_format/encoding.py new file mode 100644 index 00000000..82de4680 --- /dev/null +++ b/src/twinkle/data_format/encoding.py @@ -0,0 +1,32 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""The single definition of "is this entry already encoded model input?". + +One predicate, one place. Before this module the same rule existed three times +(``MegatronModel._not_encoded``, ``TransformersModel._not_encoded``, +``Sampler._not_encoded``), one of them carrying a comment that it was "aligned +with" another -- an invariant only a human could maintain. The wire schema needs +the same rule to decide whether an ``inputs`` entry is an ``InputFeature`` or a +``Trajectory``, so a fourth copy would have made divergence a matter of time. + +``input_embedding`` matters as much as ``input_ids``: a batch carrying only +embeddings is already encoded, and misreading it as a ``Trajectory`` sends it +through ``template.batch_encode``, which fails far away from the cause. +""" +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +# Presence of any of these keys means the entry carries encoded model input. +ENCODED_INPUT_KEYS: tuple[str, ...] = ('input_ids', 'input_embedding') + + +def is_encoded(entry: Any) -> bool: + """True when ``entry`` is an already-encoded ``InputFeature``-shaped mapping. + + A non-mapping is not encoded -- callers that need a type error raise it + themselves; this predicate answers only the classification question. + """ + if not isinstance(entry, Mapping): + return False + return any(key in entry for key in ENCODED_INPUT_KEYS) diff --git a/src/twinkle/model/megatron/megatron.py b/src/twinkle/model/megatron/megatron.py index 4e4e2bbd..f321192a 100644 --- a/src/twinkle/model/megatron/megatron.py +++ b/src/twinkle/model/megatron/megatron.py @@ -25,7 +25,7 @@ import twinkle.patch from twinkle import DeviceMesh, Platform, remote_class, remote_function, requires, torch_util from twinkle.checkpoint_engine.mixin import CheckpointEngineMixin -from twinkle.data_format import InputFeature, ModelOutput, Trajectory +from twinkle.data_format import InputFeature, ModelOutput, Trajectory, is_encoded from twinkle.hub import HubOperation from twinkle.infra import collect_tensor_dict from twinkle.loss import CrossEntropyLoss, Loss @@ -201,7 +201,7 @@ def _get_default_group(self): @staticmethod def _not_encoded(inputs): assert isinstance(inputs, dict) - return 'input_ids' not in inputs and 'input_embedding' not in inputs + return not is_encoded(inputs) @staticmethod def _slice_value_for_microbatch(value, mb_start: int, mb_end: int, micro_batch_size: int): diff --git a/src/twinkle/model/transformers/transformers.py b/src/twinkle/model/transformers/transformers.py index fcf856f8..5b8723ce 100644 --- a/src/twinkle/model/transformers/transformers.py +++ b/src/twinkle/model/transformers/transformers.py @@ -27,7 +27,7 @@ from twinkle import DeviceMesh, Platform, remote_class, remote_function from twinkle.checkpoint_engine import CheckpointEngine from twinkle.checkpoint_engine.mixin import CheckpointEngineMixin -from twinkle.data_format import InputFeature, ModelOutput, Trajectory +from twinkle.data_format import InputFeature, ModelOutput, Trajectory, is_encoded from twinkle.hub import HubOperation from twinkle.infra import collect_tensor_dict from twinkle.loss import CrossEntropyLoss, Loss @@ -395,7 +395,7 @@ def _get_default_group(self): @staticmethod def _not_encoded(inputs): assert isinstance(inputs, dict) - return 'input_ids' not in inputs and 'input_embedding' not in inputs + return not is_encoded(inputs) def _lazy_wrap_model(self): if not self._model_wrapped: diff --git a/src/twinkle/sampler/base.py b/src/twinkle/sampler/base.py index a756a93c..63fa344e 100644 --- a/src/twinkle/sampler/base.py +++ b/src/twinkle/sampler/base.py @@ -5,7 +5,7 @@ import twinkle from twinkle import remote_function -from twinkle.data_format import InputFeature, SampleResponse, SamplingParams, Trajectory +from twinkle.data_format import InputFeature, SampleResponse, SamplingParams, Trajectory, is_encoded from twinkle.patch import Patch from twinkle.template import Template from twinkle.utils import construct_class @@ -51,10 +51,11 @@ def apply_patch(self, patch_cls: Union[Patch, Type[Patch], str], **kwargs) -> No def _not_encoded(inputs: Any) -> bool: """Check if inputs are not yet encoded (i.e., is Trajectory, not InputFeature). - Aligned with TransformersModel._not_encoded for consistency. + Delegates to the single shared predicate so the three backends and the wire + schema cannot drift apart. """ assert isinstance(inputs, dict), f'Expected dict, got {type(inputs)}' - return 'input_ids' not in inputs and 'input_embedding' not in inputs + return not is_encoded(inputs) def _is_trajectory(self, inputs: Any) -> bool: """Check if inputs are Trajectory type (not encoded).""" diff --git a/src/twinkle/server/deployment.py b/src/twinkle/server/deployment.py index bf02ce76..d795d31b 100644 --- a/src/twinkle/server/deployment.py +++ b/src/twinkle/server/deployment.py @@ -39,6 +39,7 @@ from twinkle.server.telemetry.middleware import create_metrics_middleware from twinkle.server.telemetry.tracing import create_tracing_middleware from twinkle.server.utils.validation import verify_request_token +from twinkle.server.validation.errors import register_validation_error_handler from twinkle.utils.logger import get_logger from twinkle_client.types.errors import ErrorCategory, ErrorPayload @@ -94,17 +95,20 @@ def build_deployment_app( shutdown → ``on_shutdown(get_servable())`` (best-effort) then ``flush_telemetry_safely()`` so buffered OTLP batches flush on graceful replica termination; - 2. [if ``attach_cleanup_middleware``] the gateway-only lazy-cleanup + 2. the ``TwinkleServerError`` and ``RequestValidationError`` handlers, so a + rejected request body carries the same ``ErrorPayload`` shape as any other + failure; + 3. [if ``attach_cleanup_middleware``] the gateway-only lazy-cleanup middleware (registered first ⇒ innermost), since the Gateway has no per-handler hook; - 3. ``catch_unhandled_exceptions`` middleware, inside auth/tracing/metrics + 4. ``catch_unhandled_exceptions`` middleware, inside auth/tracing/metrics and outside cleanup/routes; - 4. ``verify_token`` middleware; - 5. ``create_tracing_middleware(component)``; - 6. ``create_metrics_middleware(component)``; - 7. [if ``attach_replica_id_header``] replica-id response header middleware + 5. ``verify_token`` middleware; + 6. ``create_tracing_middleware(component)``; + 7. ``create_metrics_middleware(component)``; + 8. [if ``attach_replica_id_header``] replica-id response header middleware (registered last ⇒ outermost); - 8. ``register_routes(app, get_servable)``. + 9. ``register_routes(app, get_servable)``. Args: component: ``'Gateway' | 'Model' | 'Sampler' | 'Processor'`` — used as @@ -144,6 +148,10 @@ async def lifespan(app: FastAPI): app = FastAPI(lifespan=lifespan, **(fastapi_kwargs or {})) app.add_exception_handler(TwinkleServerError, twinkle_server_error_handler) + # Request-body validation failures answer with the same ``ErrorPayload`` shape as + # every other error, registered here so all deployments behave identically rather + # than each app keeping (or forgetting) its own copy. + register_validation_error_handler(app) # Registration order matters: FastAPI runs middleware LIFO, so the LAST # registered wraps the outermost layer. Register cleanup (if any) first so diff --git a/src/twinkle/server/lifecycle/submit.py b/src/twinkle/server/lifecycle/submit.py index 4bbc7017..fe2dd4ff 100644 --- a/src/twinkle/server/lifecycle/submit.py +++ b/src/twinkle/server/lifecycle/submit.py @@ -12,65 +12,99 @@ from fastapi import Request from typing import Any -from twinkle.data_format import InputFeature, Trajectory +from twinkle.data_format import InputFeature, Trajectory, is_encoded from twinkle.server.utils.validation import get_session_id_from_request +from twinkle.server.validation import assert_request_supported +from twinkle_client.types.base import FieldRole, fields_with_role +from twinkle_client.types.data import export_batch from twinkle_client.types.lifecycle import TaskEnvelope # --------------------------------------------------------------------------- # -# Named seams. This spec implements the current semantics; the server-request-schema -# spec later replaces these function bodies without touching the shell or the return -# path, so the two specs edit disjoint regions. +# Named seams shared by every queued twinkle-native handler. # --------------------------------------------------------------------------- # def to_backend_inputs(inputs: Any, *, single: bool = False) -> Any: - """Seam A: convert raw dict/list inputs to InputFeature / Trajectory objects. - - With ``single=False`` (default) a *batch* is returned: a list of parsed objects - for a list input, a one-element list for a single dict, and the value unchanged - otherwise. With ``single=True`` exactly one parsed object is returned (the - streaming path accepts only one input): a list must contain exactly one element - or a ``ValueError`` is raised, a dict is parsed to a single object, and anything - else is passed through. Element typing is unchanged: a dict with ``input_ids`` - becomes an ``InputFeature``, otherwise a ``Trajectory``. + """Seam A: export wire-validated ``inputs`` as the objects the backend consumes. + + This is an *export*, not a validation step. The request model declares ``inputs`` + as :data:`~twinkle_client.types.data.WireInputBatch`, so a malformed batch is + already rejected during FastAPI body parsing -- before a future record exists and + before anything reaches a GPU. Validating here instead would put the first check + inside the queued task, where a rejection has already cost an enqueue. + + Entries arrive as wire models and are exported with ``exclude_none`` semantics, so + unset optional fields stay absent (Twinkle_Core branches on key presence) and + unknown keys the caller sent are preserved. ``InputFeature`` / ``Trajectory`` are + ``TypedDict``s, so constructing them is a plain dict build. + + With ``single=True`` exactly one object is returned (the streaming path accepts + only one input) and a batch of any other size is a ``ValueError``. Plain dicts pass + through unchanged: the data-plane path resolves rows itself and never goes through + the wire schema. """ + entries = export_batch(inputs) if isinstance(inputs, list) else inputs if single: - if isinstance(inputs, list): - if len(inputs) != 1: + if isinstance(entries, list): + if len(entries) != 1: raise ValueError('Streaming only supports a single input') - inputs = inputs[0] - if isinstance(inputs, dict): - return InputFeature(**inputs) if 'input_ids' in inputs else Trajectory(**inputs) - return inputs - if isinstance(inputs, list) and inputs: - first = inputs[0] - if isinstance(first, dict) and 'input_ids' in first: - return [InputFeature(**item) for item in inputs] - return [Trajectory(**item) for item in inputs] - if isinstance(inputs, dict): - if 'input_ids' in inputs: - return [InputFeature(**inputs)] - return [Trajectory(**inputs)] - return inputs + entries = entries[0] + if isinstance(entries, dict): + return _as_backend_entry(entries) + return entries + if isinstance(entries, list): + return [_as_backend_entry(entry) if isinstance(entry, dict) else entry for entry in entries] + if isinstance(entries, dict): + return [_as_backend_entry(entries)] + return entries + + +def _as_backend_entry(entry: dict[str, Any]) -> Any: + """One exported entry as its ``TypedDict`` shape.""" + return InputFeature(**entry) if is_encoded(entry) else Trajectory(**entry) def backend_kwargs(body: Any) -> dict[str, Any]: - """Seam B: the passthrough kwargs forwarded to the backend call.""" - return body.model_extra or {} + """Seam B: the keyword arguments forwarded to the backend call. + + Exactly two sources, both declared on the request model (see + :mod:`twinkle_client.types.base`): + + 1. fields whose role is ``BackendKwarg``, included iff their value is not ``None``; + 2. the contents of every ``Passthrough`` field, flattened. + + Control fields are never forwarded. That exclusion is the point of the field roles: + forwarding *all* declared non-``None`` fields would re-send ``inputs`` / + ``adapter_name`` / ``seq_id``, which the handlers already pass explicitly -- a + duplicate keyword argument at best, and a protocol field leaking into a backend + signature at worst. + """ + model_cls = type(body) + kwargs: dict[str, Any] = {} + for name in fields_with_role(model_cls, FieldRole.BackendKwarg): + value = getattr(body, name, None) + if value is not None: + kwargs[name] = value + for name in fields_with_role(model_cls, FieldRole.Passthrough): + region = getattr(body, name, None) or {} + overlap = set(region) & set(kwargs) + if overlap: + raise ValueError(f'{name} collides with declared backend parameters: {", ".join(sorted(overlap))}') + kwargs.update(region) + return kwargs def input_metrics(self, body: Any, *, data_parallel: bool = False) -> dict[str, Any]: """Seam C: scheduling metrics (input_tokens, and batch_size/data_world_size). - Defensive shape (isinstance guards + .get defaults) because the body is not yet - strictly validated; a non-dict element must not raise here. + Reads validated wire models, so no isinstance guards: ``inputs`` is a list and + ``input_ids`` is either absent or a list of ints. """ inputs = body.inputs - inputs_list = inputs if isinstance(inputs, list) else [inputs] - input_tokens = sum(len(inp.get('input_ids', [])) if isinstance(inp, dict) else 0 for inp in inputs_list) + input_tokens = sum(len(getattr(entry, 'input_ids', None) or ()) for entry in inputs) metrics: dict[str, Any] = {'input_tokens': input_tokens} if data_parallel: - metrics['batch_size'] = len(inputs_list) + metrics['batch_size'] = len(inputs) metrics['data_world_size'] = self.data_world_size return metrics @@ -92,13 +126,23 @@ async def run_submit( backend_call: Callable[..., Coroutine], metrics: Callable[[Any, Any], dict[str, Any]] | None = None, assert_resource: bool = True, + capability: str | None = None, ) -> TaskEnvelope: """The common Submit_Endpoint judgment sequence, called by every queued twinkle-native handler instead of being repeated in each. - Order is load-bearing: request start -> adapter resolution -> ``submit_and_peek`` - (whose ``schedule_task`` runs preflight). Every admission check runs before any - state write, so a rejected request writes nothing. + Order is load-bearing: request start -> adapter resolution -> preflight -> + ``submit_and_peek`` (whose ``schedule_task`` runs its own resource preflight). + Every admission check runs before any state write, so a rejected request writes + nothing. + + ``assert_request_supported`` is the one place the request is checked against *this + deployment*: a parameter that only exists on the other backend and an endpoint this + backend does not implement are decided here -- before the seq claim and before the + enqueue, so a rejected request runs on zero data-parallel ranks. Putting these checks + in the queued task instead would let an incompatible request cost a full GPU dispatch. + Passthrough keys are forwarded unjudged (no spelling check): see + :mod:`twinkle.server.validation.backend_compat`. A plain helper, not a signature-rewriting decorator: each handler keeps its natural FastAPI signature so the app stays shallow enough for Ray Serve to cloudpickle (a @@ -117,10 +161,16 @@ async def run_submit( and returns the JSON-safe task result. ``metrics(self, body)`` supplies scheduling kwargs; omit it for control-plane ops. ``assert_resource`` guards on the adapter existing before the work runs; set it False for endpoints that create/drop it. + ``capability`` names the backend capability the endpoint needs, when the endpoint is + not implemented by every backend. """ token = await self._on_request_start(request) adapter_name = resolve_twinkle_adapter_name(request, body.adapter_name) + # ---- Preflight: decidable from the body plus this deployment's backend, so it + # runs before any state write and before the enqueue. ---- + assert_request_supported(self, body, capability=capability) + schedule_kwargs = metrics(self, body) if metrics is not None else {} async def _task(): diff --git a/src/twinkle/server/model/twinkle_handlers.py b/src/twinkle/server/model/twinkle_handlers.py index 993ed214..8495c745 100644 --- a/src/twinkle/server/model/twinkle_handlers.py +++ b/src/twinkle/server/model/twinkle_handlers.py @@ -28,6 +28,7 @@ from twinkle.server.model.utils import (data_plane_request_shape, merge_forward_kwargs, resolve_data_plane_model_inputs, select_output_rows) from twinkle.server.utils.validation import get_session_id_from_request +from twinkle.server.validation import BackendCapability from twinkle.utils.logger import get_logger logger = get_logger() @@ -90,7 +91,14 @@ async def _call(self, body, adapter_name, token): **backend_kwargs(body)) return {'result': ret} - return await run_submit(self, request, body, task_type='forward', backend_call=_call, metrics=_dp_metrics) + return await run_submit( + self, + request, + body, + task_type='forward', + backend_call=_call, + metrics=_dp_metrics, + capability=BackendCapability.Forward) @app.post('/twinkle/forward_only', response_model=types.TaskEnvelope) async def forward_only( @@ -110,7 +118,8 @@ async def _call(self, body, adapter_name, token): @app.post('/twinkle/forward_backward', response_model=types.TaskEnvelope) async def forward_backward( - request: Request, body: types.ForwardRequest, self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: + request: Request, body: types.ForwardBackwardTaskRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: async def _call(self, body, adapter_name, token): @@ -147,7 +156,13 @@ async def _call(self, body, adapter_name, token): **backend_kwargs(body)) return {'result': ret} - return await run_submit(self, request, body, task_type='calculate_loss', backend_call=_call) + return await run_submit( + self, + request, + body, + task_type='calculate_loss', + backend_call=_call, + capability=BackendCapability.CalculateLoss) @app.post('/twinkle/backward', response_model=types.TaskEnvelope) async def backward(request: Request, body: types.AdapterRequest, @@ -157,7 +172,8 @@ async def _call(self, body, adapter_name, token): await self.call_backend( self.model.backward, adapter_name=self.resolve_model_adapter_name(adapter_name), **backend_kwargs(body)) - return await run_submit(self, request, body, task_type='backward', backend_call=_call) + return await run_submit( + self, request, body, task_type='backward', backend_call=_call, capability=BackendCapability.Backward) # ------------------------------------------------------------------ # # Data-plane forward family (DataRef inputs; response only enters the contract) @@ -170,7 +186,7 @@ async def forward_from_data_plane( async def _call(self, body, adapter_name, token): raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) - kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) + kwargs = merge_forward_kwargs(backend_kwargs(body), field_kwargs) ret = await self.call_backend( self.model.forward, inputs=to_backend_inputs(raw_inputs), @@ -179,7 +195,13 @@ async def _call(self, body, adapter_name, token): return {'result': ret} return await run_submit( - self, request, body, task_type='forward_from_data_plane', backend_call=_call, metrics=_data_plane_metrics) + self, + request, + body, + task_type='forward_from_data_plane', + backend_call=_call, + metrics=_data_plane_metrics, + capability=BackendCapability.Forward) @app.post('/twinkle/forward_only_from_data_plane', response_model=types.TaskEnvelope) async def forward_only_from_data_plane( @@ -189,7 +211,7 @@ async def forward_only_from_data_plane( async def _call(self, body, adapter_name, token): raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) inputs = to_backend_inputs(raw_inputs) - kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) + kwargs = merge_forward_kwargs(backend_kwargs(body), field_kwargs) ret = await self.call_backend( self.model.forward_only, inputs=inputs, @@ -216,7 +238,7 @@ async def forward_backward_from_data_plane( async def _call(self, body, adapter_name, token): raw_inputs, field_kwargs = await resolve_data_plane_model_inputs(body, self.data_plane) - kwargs = merge_forward_kwargs(body.model_extra or {}, field_kwargs) + kwargs = merge_forward_kwargs(backend_kwargs(body), field_kwargs) ret = await self.call_backend( self.model.forward_backward, inputs=to_backend_inputs(raw_inputs), @@ -238,11 +260,14 @@ async def _call(self, body, adapter_name, token): @app.post('/twinkle/clip_grad_norm', response_model=types.TaskEnvelope) async def clip_grad_norm( - request: Request, body: types.AdapterRequest, self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: + request: Request, body: types.ClipGradNormRequest, + self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: async def _call(self, body, adapter_name, token): ret = await self.call_backend( self.model.clip_grad_norm, + max_grad_norm=body.max_grad_norm, + norm_type=body.norm_type, adapter_name=self.resolve_model_adapter_name(adapter_name), **backend_kwargs(body)) return {'result': str(ret)} @@ -250,7 +275,7 @@ async def _call(self, body, adapter_name, token): return await run_submit(self, request, body, task_type='clip_grad_norm', backend_call=_call) @app.post('/twinkle/step', response_model=types.TaskEnvelope) - async def step(request: Request, body: types.AdapterRequest, + async def step(request: Request, body: types.StepRequest, self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: async def _call(self, body, adapter_name, token): @@ -272,7 +297,7 @@ async def _call(self, body, adapter_name, token): return await run_submit(self, request, body, task_type='zero_grad', backend_call=_call) @app.post('/twinkle/lr_step', response_model=types.TaskEnvelope) - async def lr_step(request: Request, body: types.AdapterRequest, + async def lr_step(request: Request, body: types.LrStepRequest, self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: async def _call(self, body, adapter_name, token): @@ -549,7 +574,7 @@ async def add_adapter_to_model( async def _task(): from peft import LoraConfig - extra_kwargs = body.model_extra or {} + extra_kwargs = backend_kwargs(body) training_run_manager = create_training_run_manager(token, client_type='twinkle') lora_config = None if isinstance(config, LoraConfig): diff --git a/src/twinkle/server/processor/twinkle_handlers.py b/src/twinkle/server/processor/twinkle_handlers.py index e8f8792c..58fd52d2 100644 --- a/src/twinkle/server/processor/twinkle_handlers.py +++ b/src/twinkle/server/processor/twinkle_handlers.py @@ -44,7 +44,7 @@ async def create( processor_type_name = body.processor_type class_type = body.class_type - _kwargs = body.model_extra or {} + _kwargs = dict(body.init_kwargs) assert processor_type_name in _PROCESSOR_TYPES, f'Invalid processor type: {processor_type_name}' processor_module = importlib.import_module(f'twinkle.{processor_type_name}') @@ -98,7 +98,7 @@ async def call( processor_id = body.processor_id function_name = body.function - _kwargs = body.model_extra or {} + _kwargs = dict(body.call_kwargs) processor_id = processor_id[4:] self.assert_resource_exists(processor_id) processor = self.resource_dict.get(processor_id) diff --git a/src/twinkle/server/sampler/twinkle_handlers.py b/src/twinkle/server/sampler/twinkle_handlers.py index 0d85c2f5..c50e9961 100644 --- a/src/twinkle/server/sampler/twinkle_handlers.py +++ b/src/twinkle/server/sampler/twinkle_handlers.py @@ -21,9 +21,10 @@ import numpy as np import twinkle_client.types as types +from twinkle_client.types import sampler as sampler_types from twinkle.data_format import SamplingParams from twinkle.server.exceptions import RequestRejectedError -from twinkle.server.lifecycle.submit import resolve_twinkle_adapter_name, to_backend_inputs +from twinkle.server.lifecycle.submit import backend_kwargs, resolve_twinkle_adapter_name, to_backend_inputs from twinkle.server.sampler.weights import resolve_sampler_weights from twinkle.server.telemetry.correlation import MODEL_ID from twinkle.server.telemetry.tracing import traced_operation @@ -214,10 +215,10 @@ def _register_twinkle_sampler_routes(app: FastAPI, self_fn: Callable[[], Sampler It is wired in via Depends so it is resolved lazily at request time. """ - @app.post('/twinkle/create', response_model=types.CreateResponse) - async def create(request: Request, self: SamplerManagement = Depends(self_fn)) -> types.CreateResponse: + @app.post('/twinkle/create', response_model=sampler_types.SamplerCreateResponse) + async def create(request: Request, self: SamplerManagement = Depends(self_fn)) -> sampler_types.SamplerCreateResponse: """Health check / session creation endpoint.""" - return types.CreateResponse() + return sampler_types.SamplerCreateResponse() @app.post('/twinkle/sample', response_model=types.TaskEnvelope) async def sample(request: Request, body: types.SampleRequest, @@ -260,9 +261,9 @@ async def _task(): ) return types.SampleResponseModelList(samples=_to_sample_response_models(responses)).model_dump() - # Calculate metrics for queue scheduling - inputs_list = body.inputs if isinstance(body.inputs, list) else [body.inputs] - input_tokens = sum(len(inp.get('input_ids', [])) if isinstance(inp, dict) else 0 for inp in inputs_list) + # Calculate metrics for queue scheduling. The body is wire-validated, so the + # entries are models and ``input_ids`` is absent or a list of ints. + input_tokens = sum(len(getattr(entry, 'input_ids', None) or ()) for entry in body.inputs) return await self.submit_and_peek(_task, token=token, input_tokens=input_tokens, task_type='sample') @app.post('/twinkle/sample_to_data_plane', response_model=types.TaskEnvelope) @@ -340,24 +341,23 @@ async def unload_adapter_paths( await self.call_backend(unload, resolved_paths) return {'status': 'ok'} - @app.post('/twinkle/set_template', response_model=types.SetTemplateResponse) + @app.post('/twinkle/set_template', response_model=sampler_types.SamplerSetTemplateResponse) async def set_template( request: Request, - body: types.SetTemplateRequest, + body: sampler_types.SamplerSetTemplateRequest, self: SamplerManagement = Depends(self_fn), - ) -> types.SetTemplateResponse: + ) -> sampler_types.SamplerSetTemplateResponse: """Set the chat template for encoding Trajectory inputs.""" - extra_kwargs = body.model_extra or {} with traced_operation('sampler.set_template'): - await self.call_backend(self.sampler.set_template, body.template_cls, **extra_kwargs) - return types.SetTemplateResponse() + await self.call_backend(self.sampler.set_template, body.template_cls, **backend_kwargs(body)) + return sampler_types.SamplerSetTemplateResponse() - @app.post('/twinkle/add_adapter_to_sampler', response_model=types.AddAdapterResponse) + @app.post('/twinkle/add_adapter_to_sampler', response_model=sampler_types.SamplerAddAdapterResponse) async def add_adapter_to_sampler( request: Request, - body: types.AddAdapterRequest, + body: sampler_types.SamplerAddAdapterRequest, self: SamplerManagement = Depends(self_fn), - ) -> types.AddAdapterResponse: + ) -> sampler_types.SamplerAddAdapterResponse: """Add a LoRA adapter to the sampler.""" # Raised, not asserted: decidable from the request body alone, so it owes the caller # a real 400 rather than an AssertionError surfacing as a 500 -- and a bare assert @@ -372,7 +372,7 @@ async def add_adapter_to_sampler( with traced_operation('sampler.add_adapter_to_sampler', attrs={MODEL_ID: self.model_id}): await self.call_backend(self.sampler.add_adapter_to_sampler, full_adapter_name, config) - return types.AddAdapterResponse(adapter_name=full_adapter_name) + return sampler_types.SamplerAddAdapterResponse(adapter_name=full_adapter_name) @app.post('/twinkle/apply_patch') async def apply_patch( @@ -381,10 +381,9 @@ async def apply_patch( self: SamplerManagement = Depends(self_fn), ) -> None: from twinkle_client.common.serialize import deserialize_object - extra_kwargs = body.model_extra or {} patch_cls = deserialize_object(body.patch_cls) with traced_operation('sampler.apply_patch'): - await self.call_backend(self.sampler.apply_patch, patch_cls, **extra_kwargs) + await self.call_backend(self.sampler.apply_patch, patch_cls, **backend_kwargs(body)) @app.post('/twinkle/sample_stream') async def sample_stream( diff --git a/src/twinkle/server/validation/__init__.py b/src/twinkle/server/validation/__init__.py new file mode 100644 index 00000000..5a9afdc4 --- /dev/null +++ b/src/twinkle/server/validation/__init__.py @@ -0,0 +1,32 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Request checks that are decidable before a task is enqueued. + +Two checks behind one entry point (:func:`assert_request_supported`), both running in +the synchronous request path: + +- the endpoint exists on this deployment's backend (else 501); +- no declared parameter belongs to a different backend (else 422). + +Their shared property is what makes them belong together: each is answerable from the +request body plus this deployment's configuration, so a rejection costs one HTTP round +trip and runs the backend method on **zero** data-parallel ranks. Discovering the same +problems from a backend exception instead means the failure surfaces inside the NCCL +critical section, where some ranks have already done work. + +Both read *declared* metadata, so neither can reject a valid request. A third, +heuristic check over passthrough key spellings was implemented and removed for failing +that bar -- see :mod:`.backend_compat` for the case that killed it. + +:mod:`.errors` is the other half of the story: it gives FastAPI's own body-validation +failures the same wire shape as these, so a caller sees one error format. +""" +from .backend_compat import BackendCapability, EndpointUnavailableError, assert_request_supported, resolve_backend +from .errors import register_validation_error_handler + +__all__ = [ + 'BackendCapability', + 'EndpointUnavailableError', + 'assert_request_supported', + 'register_validation_error_handler', + 'resolve_backend', +] diff --git a/src/twinkle/server/validation/backend_compat.py b/src/twinkle/server/validation/backend_compat.py new file mode 100644 index 00000000..1986a9df --- /dev/null +++ b/src/twinkle/server/validation/backend_compat.py @@ -0,0 +1,125 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Reject a request this deployment cannot serve, before it is enqueued. + +Two questions, both answerable from the request body plus the deployment's own +configuration: + +1. does the body set a parameter that only exists on the *other* backend? +2. does the endpoint exist on this backend at all? + +Both used to be answered by the backend raising during execution -- which means after +the task was enqueued, a future record written, and the call fanned out to every +data-parallel rank. Answering them here makes the number of ranks that ran the backend +method for a rejected request exactly zero. + +**Why there is no spelling check over passthrough keys.** One was implemented here and +removed after it rejected a working call: ``set_processor('InputProcessor', +padding_side='right')``, where ``padding_side`` is a real parameter that +``InputProcessor`` reads via ``kwargs.get('padding_side')`` and therefore never declares. +It scored 0.75 against the declared ``padding_free``. That is not a tunable threshold +problem -- ``inspect.signature`` cannot see a ``**kwargs`` read at all, so "misspelled" +and "read out of ``**kwargs``" are indistinguishable to it, and any threshold catching +``bate`` -> ``beta`` also catches this. A check that rejects valid requests is worse than +no check, so the passthrough region is forwarded as given and a misspelled plugin +argument still surfaces from the plugin itself. +""" +from __future__ import annotations + +from enum import StrEnum +from typing import Any, Optional + +from twinkle.server.exceptions import RequestRejectedError, TwinkleServerError +from twinkle_client.types.base import FieldRole, fields_with_role, read_backend_only +from twinkle_client.types.errors import ErrorCategory + + +class BackendCapability(StrEnum): + """An endpoint that not every backend implements. + + Only the gradient-path splits belong here. Megatron fuses forward and backward, so + it raises ``NotImplementedError('Megatron only supports forward_backward and + forward_only')`` for each of these three; naming them makes that a 501 with the + alternative endpoints in the message instead of a 500 carrying a backend traceback. + """ + + Forward = 'forward' + Backward = 'backward' + CalculateLoss = 'calculate_loss' + + +# Capabilities a backend does NOT provide. Absent backends support everything; ``mock`` +# is deliberately absent because it is a test double that accepts every call. +_UNSUPPORTED: dict[str, frozenset[str]] = { + 'megatron': frozenset({BackendCapability.Forward, BackendCapability.Backward, BackendCapability.CalculateLoss}), +} + +_ALTERNATIVES = 'use `forward_backward` (training) or `forward_only` (inference) instead' + + +class EndpointUnavailableError(TwinkleServerError): + """The endpoint is not implemented by this deployment's backend. + + 501, not 4xx: the request is well-formed and the caller is not at fault -- this + deployment simply cannot serve it. ``Server`` category, and no traceback, because + the answer is a deployment fact rather than a crash. + """ + + error_code = 501 + category = ErrorCategory.Server + + +def resolve_backend(service: Any) -> Optional[str]: + """This deployment's declared backend, or ``None`` when it has no backend concept. + + Read from the deployment's own configuration (``ModelManagement.backend``), never + inferred from the wrapper class name, the request body, or a backend exception: those + are all restatements of the same fact one step further from the source, and the + sampler deployment has no ``backend`` at all. + """ + backend = getattr(service, 'backend', None) + return backend if isinstance(backend, str) else None + + +def assert_endpoint_available(service: Any, capability: Optional[str]) -> None: + """Raise 501 when this deployment's backend does not implement ``capability``.""" + if capability is None: + return + backend = resolve_backend(service) + if backend is not None and capability in _UNSUPPORTED.get(backend, frozenset()): + raise EndpointUnavailableError(f'`{capability}` is not available on the {backend} backend; {_ALTERNATIVES}.') + + +def assert_backend_fields(service: Any, body: Any) -> None: + """Raise 422 for a field restricted to a backend other than this deployment's. + + Only a field carrying a non-``None`` value is checked. A restricted field is always + ``Optional[...] = None``, precisely so that "not sent" and "sent to the wrong + backend" stay distinguishable -- were it given the backend's own default, every + request on the other half of the fleet would be rejected. + """ + backend = resolve_backend(service) + if backend is None or backend == 'mock': + return + offenders = [] + for name, info in fields_with_role(type(body), FieldRole.BackendKwarg).items(): + allowed = read_backend_only(info) + if allowed and backend not in allowed and getattr(body, name, None) is not None: + offenders.append((name, allowed)) + if offenders: + details = '; '.join(f'`{name}` is only supported on {"/".join(allowed)}' for name, allowed in offenders) + raise RequestRejectedError( + f'This deployment runs the {backend} backend. {details}. Remove the parameter or target a ' + f'deployment running a supporting backend.', + error_code=422) + + +def assert_request_supported(service: Any, body: Any, *, capability: Optional[str] = None) -> None: + """The single preflight entry point, called from ``run_submit`` before the enqueue. + + Both checks read *declared* metadata, so neither can produce a false positive. A + spelling heuristic over passthrough keys was tried here and removed: signature + reflection cannot distinguish a misspelling from a parameter a target reads straight + out of ``**kwargs``, so it rejected working calls. See the module docstring. + """ + assert_endpoint_available(service, capability) + assert_backend_fields(service, body) diff --git a/src/twinkle/server/validation/errors.py b/src/twinkle/server/validation/errors.py new file mode 100644 index 00000000..895c7eca --- /dev/null +++ b/src/twinkle/server/validation/errors.py @@ -0,0 +1,77 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Deliver a request-body validation failure in the same shape as every other error. + +FastAPI's default handler answers ``RequestValidationError`` with +``{"detail": [...]}``, which is a second error shape on the wire: a client that learned +to read ``error`` / ``category`` / ``error_code`` from :class:`ErrorPayload` -- what +every other twinkle failure uses -- gets nothing it recognises from a 422. Registering +this on the shared app builder makes the Model, Sampler and Processor deployments answer +identically. + +The per-field ``details`` are the point of a 422: they name the offending field, its +path inside the body, and why it was rejected, so a caller can fix the request without +guessing. There is no traceback -- a rejected body is the caller's problem, not a crash, +and pydantic's error list already localises it exactly. +""" +from __future__ import annotations + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from typing import Any + +from twinkle_client.types.errors import ErrorCategory, ErrorPayload + +# A body can produce hundreds of errors (one per element of a mis-typed tensor), and a +# response listing all of them helps nobody while costing bandwidth on every retry. +_MAX_DETAILS = 20 + + +def _detail(error: dict[str, Any]) -> dict[str, Any]: + """One pydantic error as a JSON-safe detail entry.""" + location = [str(part) for part in error.get('loc', ())] + return { + 'field': location[-1] if location else '', + 'path': '.'.join(location), + 'type': error.get('type', ''), + 'message': error.get('msg', ''), + } + + +def _summary(errors: list[dict[str, Any]]) -> str: + fields = [] + for error in errors: + path = '.'.join(str(part) for part in error.get('loc', ())) + if path and path not in fields: + fields.append(path) + shown = ', '.join(fields[:_MAX_DETAILS]) or 'request body' + suffix = '' if len(fields) <= _MAX_DETAILS else f' (+{len(fields) - _MAX_DETAILS} more)' + return f'Request body validation failed for: {shown}{suffix}' + + +def _mentions_unknown_field(errors: list[dict[str, Any]]) -> bool: + return any(error.get('type') == 'extra_forbidden' for error in errors) + + +async def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse: + """Map a body validation failure to a 422 carrying an ``ErrorPayload``.""" + errors = list(exc.errors()) + message = _summary(errors) + if _mentions_unknown_field(errors): + # An unknown top-level field is what an older client looks like against a newer + # server, so say so instead of leaving the caller to infer it from a field list. + message += ('. Unknown fields are rejected; if this worked before, upgrade ' + 'twinkle-kit on the client to match the server version.') + payload = ErrorPayload( + error=message[:1024], + category=ErrorCategory.User, + error_code=422, + request_id=getattr(request.state, 'request_id', None) or '', + details=[_detail(error) for error in errors[:_MAX_DETAILS]], + ) + return JSONResponse(status_code=422, content=payload.model_dump(mode='json', exclude_none=True)) + + +def register_validation_error_handler(app: FastAPI) -> None: + """Install the handler on one deployment app.""" + app.add_exception_handler(RequestValidationError, validation_error_handler) diff --git a/src/twinkle_client/_request_builder.py b/src/twinkle_client/_request_builder.py new file mode 100644 index 00000000..1396cac4 --- /dev/null +++ b/src/twinkle_client/_request_builder.py @@ -0,0 +1,110 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Build a twinkle-native request body from a caller's keyword arguments. + +Every public client method used to hand-assemble a ``json_data`` dict. That made the +request schema a second, implicit source of truth: a field the server declared but the +client never sent, or a name only one side spelled correctly, was invisible until a +request failed on the wire. Here the request *model* is the only source of truth -- +the client instantiates it, so the same rules that guard the server also guard the +caller, in-process and without a round trip. + +Routing a caller's ``**kwargs`` needs exactly three rules, all read off the model: + +1. the name is a declared field -> assign it to that field; +2. it is not declared and the model has exactly one passthrough region -> put it there; +3. it is not declared and the model has none, or more than one -> raise. + +Rule 3 does not guess. A processor call has both ``init_kwargs`` (the constructor's +arguments) and ``call_kwargs`` (the invoked method's); no rule based on the name alone +can tell which one a caller meant, and choosing wrong sends a valid argument to the +wrong callable -- a silently wrong result rather than an error. +""" +from __future__ import annotations + +from pydantic import BaseModel +from typing import Any, Mapping + +from twinkle_client.common.json_utils import json_safe +from twinkle_client.exceptions import TwinkleClientValidationError +from twinkle_client.types.base import FieldRole, fields_with_role + + +def to_wire_value(value: Any) -> Any: + """Convert one caller-supplied argument to a JSON-native value. + + Handles the three object kinds the client has always accepted in a request body: + a server-side component handle (sent as its id), a ``DatasetMeta`` / ``LoraConfig`` + (sent as the canonical serialized form the server decodes), and numpy / torch + values (sent as nested lists). + + Anything else is passed through for the model to validate, so an unsupported type + is reported by pydantic with its field path instead of by a generic error here. + """ + if hasattr(value, 'processor_id'): + return value.processor_id + from twinkle.dataset import DatasetMeta + from peft import LoraConfig + if isinstance(value, (DatasetMeta, LoraConfig)): + from twinkle_client.common.serialize import serialize_object + return serialize_object(value) + if isinstance(value, BaseModel): + return value.model_dump(mode='json') + return json_safe(value) + + +def build_request(model_cls: type[BaseModel], /, **values: Any) -> BaseModel: + """Instantiate ``model_cls`` from caller arguments, routing undeclared names. + + ``None`` values for undeclared names are dropped rather than routed: client methods + pass optional arguments unconditionally, and forwarding an explicit ``None`` into a + passthrough region would hand the backend a null it never had before. + + Raises: + TwinkleClientValidationError: an argument has no field and no unambiguous + passthrough region, or it collides with an explicitly passed region key. + pydantic.ValidationError: the assembled body violates the model. + """ + declared = model_cls.model_fields + regions = fields_with_role(model_cls, FieldRole.Passthrough) + + body: dict[str, Any] = {} + routed: dict[str, Any] = {} + for name, value in values.items(): + if name in declared: + body[name] = to_wire_value(value) if value is not None else None + elif value is None: + continue + elif len(regions) == 1: + routed[name] = to_wire_value(value) + elif not regions: + raise TwinkleClientValidationError( + f'{model_cls.__name__} has no field {name!r} and no passthrough region to put it in; ' + f'known fields: {sorted(declared)}') + else: + raise TwinkleClientValidationError( + f'{model_cls.__name__} has no field {name!r} and more than one passthrough region, so its ' + f'target is ambiguous. Pass it inside one of: {sorted(regions)}') + + if routed: + region = next(iter(regions)) + explicit = body.get(region) or {} + if not isinstance(explicit, Mapping): + raise TwinkleClientValidationError(f'{region} must be a mapping, got {type(explicit).__name__}') + collisions = set(explicit) & set(routed) + if collisions: + raise TwinkleClientValidationError( + f'these arguments were passed both directly and inside {region}: {sorted(collisions)}') + body[region] = {**explicit, **routed} + + return model_cls(**body) + + +def request_json(body: BaseModel) -> str: + """Serialize a request body once. + + ``exclude_none=True`` keeps unset optionals off the wire, which is what lets the + server treat "absent" and "not requested" as the same thing rather than maintaining + a second set of defaults. One pydantic-core pass, not a Python-level walk followed + by ``json.dumps``. + """ + return body.model_dump_json(exclude_none=True) diff --git a/src/twinkle_client/common/__init__.py b/src/twinkle_client/common/__init__.py new file mode 100644 index 00000000..930ef1ba --- /dev/null +++ b/src/twinkle_client/common/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Client-internal helpers shared across the twinkle_client subpackages. + +Regular package (carries this ``__init__``) so the modules here -- ``component_rpc``, +``serialize``, ``json_utils`` -- are included by ``setuptools.packages.find`` in a +built wheel; a namespace-only directory would be dropped from the distribution. +""" diff --git a/src/twinkle_client/common/component_rpc.py b/src/twinkle_client/common/component_rpc.py new file mode 100644 index 00000000..e34c5494 --- /dev/null +++ b/src/twinkle_client/common/component_rpc.py @@ -0,0 +1,53 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Shared plumbing for the processor-backed component clients. + +Dataset / dataloader / processor wrappers all talk to the same two generic endpoints, +so they share one place that builds those two request bodies. Before this, each method +spread its arguments over the top level of a hand-built dict, which made the envelope +indistinguishable from its payload -- a misspelt ``processor_id`` was just another +argument. Nesting the payload under the declared passthrough region is what lets the +envelope be strict. +""" +from __future__ import annotations + +from typing import Any + +from twinkle_client._request_builder import build_request +from twinkle_client.http import get_base_url, http_post_model +from twinkle_client.types.processor import (ProcessorCallRequest, ProcessorCallResponse, ProcessorCreateRequest, + ProcessorCreateResponse) + +# Sentinel: "caller did not pass an HTTP timeout", so the shared default applies. A +# literal ``None`` means "no timeout at all" in ``http_post_model``, so it cannot double +# as the unset marker. +_UNSET = object() + + +def processor_base_url() -> str: + """The single processor route prefix used by every component client.""" + return f'{get_base_url()}/processor/twinkle' + + +def create_remote_component(processor_type: str, class_type: str, **init_kwargs: Any) -> str: + """Create a server-side component and return its ``pid:``-prefixed id.""" + body = build_request( + ProcessorCreateRequest, processor_type=processor_type, class_type=class_type, **init_kwargs) + response = http_post_model(f'{processor_base_url()}/create', body) + return ProcessorCreateResponse(**response.json()).processor_id + + +def call_remote_component(processor_id: str, function: str, http_timeout: Any = _UNSET, /, **call_kwargs: Any) -> Any: + """Invoke ``function`` on a server-side component and return its result. + + ``http_timeout`` is positional-only so it can never be mistaken for -- or collide + with -- one of the remote callable's own arguments, which are all keywords. + + ``StopIteration`` propagates from the HTTP layer on an exhausted iterator (the + server answers 410), which is what makes a remote ``__next__`` usable in a plain + ``for`` loop. + """ + body = build_request(ProcessorCallRequest, processor_id=processor_id, function=function, **call_kwargs) + url = f'{processor_base_url()}/call' + response = ( + http_post_model(url, body) if http_timeout is _UNSET else http_post_model(url, body, timeout=http_timeout)) + return ProcessorCallResponse(**response.json()).result diff --git a/src/twinkle_client/dataloader/dataloader.py b/src/twinkle_client/dataloader/dataloader.py index 86e2bbf4..f9798501 100644 --- a/src/twinkle_client/dataloader/dataloader.py +++ b/src/twinkle_client/dataloader/dataloader.py @@ -1,115 +1,37 @@ - +# Copyright (c) ModelScope Contributors. All rights reserved. from typing import Callable, Type, Union -from twinkle_client.http import http_post + from twinkle.dataset import Dataset from twinkle.processor import InputProcessor +from twinkle_client.common.component_rpc import call_remote_component, create_remote_component + class DataLoader(object): """Client wrapper for DataLoader that calls server HTTP endpoints.""" def __init__(self, dataset: Union[Dataset, Callable], **kwargs): - from twinkle_client.http import get_base_url + self.processor_id = create_remote_component('dataloader', 'DataLoader', dataset=dataset, **kwargs) - self.server_url = f'{get_base_url()}/processor/twinkle' - response = http_post( - url=f'{self.server_url}/create', - json_data={ - 'processor_type': 'dataloader', - 'class_type': 'DataLoader', - **{'dataset': dataset}, **kwargs - } - ) - response.raise_for_status() - self.processor_id = response.json()['processor_id'] - - def __len__(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__len__', - **{}, - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, '__len__') def set_processor(self, processor_cls: Union[Type[InputProcessor], str, InputProcessor, Callable], **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'set_processor', - **{'processor_cls': processor_cls}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'set_processor', processor_cls=processor_cls, **kwargs) def __iter__(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__iter__', - **{}, - } - ) - response.raise_for_status() + call_remote_component(self.processor_id, '__iter__') return self - + def __next__(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__next__', - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, '__next__') def skip_consumed_samples(self, consumed_train_samples: int): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'skip_consumed_samples', - **{'consumed_train_samples': consumed_train_samples}, - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component( + self.processor_id, 'skip_consumed_samples', consumed_train_samples=consumed_train_samples) def resume_from_checkpoint(self, consumed_train_samples, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'resume_from_checkpoint', - **{'consumed_train_samples': consumed_train_samples}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component( + self.processor_id, 'resume_from_checkpoint', consumed_train_samples=consumed_train_samples, **kwargs) def get_state(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'get_state', - **{}, - } - ) - response.raise_for_status() - return response.json()["result"] - \ No newline at end of file + return call_remote_component(self.processor_id, 'get_state') diff --git a/src/twinkle_client/dataset/base.py b/src/twinkle_client/dataset/base.py index bec2d430..f52bf3ab 100644 --- a/src/twinkle_client/dataset/base.py +++ b/src/twinkle_client/dataset/base.py @@ -1,191 +1,83 @@ - +# Copyright (c) ModelScope Contributors. All rights reserved. from typing import Any, Callable, Dict, Optional, Type, Union -from twinkle_client.http import http_post -from twinkle.dataset import Dataset + from twinkle.dataset import DatasetMeta -from twinkle.preprocessor import DataFilter -from twinkle.preprocessor import Preprocessor +from twinkle.preprocessor import DataFilter, Preprocessor from twinkle.template import Template +from twinkle_client.common.component_rpc import call_remote_component, create_remote_component + class Dataset(object): """Client wrapper for Dataset that calls server HTTP endpoints.""" def __init__(self, dataset_meta: DatasetMeta = None, **kwargs): - from twinkle_client.http import get_base_url - - self.server_url = f'{get_base_url()}/processor/twinkle' - response = http_post( - url=f'{self.server_url}/create', - json_data={ - 'processor_type': 'dataset', - 'class_type': 'Dataset', - **{'dataset_meta': dataset_meta}, **kwargs - } - ) - response.raise_for_status() - self.processor_id = response.json()['processor_id'] - - + self.processor_id = create_remote_component('dataset', 'Dataset', dataset_meta=dataset_meta, **kwargs) + def set_template(self, template_func: Union[Template, Type[Template], str], **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'set_template', - **{'template_func': template_func}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'set_template', template_func=template_func, **kwargs) def encode(self, add_generation_prompt: bool = False, timeout: Optional[int] = 600, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'encode', - **{'add_generation_prompt': add_generation_prompt}, - **kwargs - }, - timeout=timeout - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component( + self.processor_id, 'encode', timeout, add_generation_prompt=add_generation_prompt, **kwargs) def check(self, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'check', - **{}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'check', **kwargs) def cast_column(self, column: str, decode: bool = True): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'cast_column', - **{'column': column, 'decode': decode}, - } - ) - response.raise_for_status() - return response.json()["result"] - - - def map(self, preprocess_func: Union[Preprocessor, Callable, str, Type[Preprocessor]], dataset_meta: DatasetMeta = None, init_args: Dict[str, Any] = None, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'map', - **{'preprocess_func': preprocess_func, 'dataset_meta': dataset_meta, 'init_args': init_args}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - - - def filter(self, filter_func: Union[Callable, str, Type[DataFilter], DataFilter], dataset_meta: DatasetMeta = None, init_args: Dict[str, Any] = None, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'filter', - **{'filter_func': filter_func, 'dataset_meta': dataset_meta, 'init_args': init_args}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'cast_column', column=column, decode=decode) + + def map(self, + preprocess_func: Union[Preprocessor, Callable, str, Type[Preprocessor]], + dataset_meta: DatasetMeta = None, + init_args: Dict[str, Any] = None, + **kwargs): + return call_remote_component( + self.processor_id, + 'map', + preprocess_func=preprocess_func, + dataset_meta=dataset_meta, + init_args=init_args, + **kwargs) + + def filter(self, + filter_func: Union[Callable, str, Type[DataFilter], DataFilter], + dataset_meta: DatasetMeta = None, + init_args: Dict[str, Any] = None, + **kwargs): + return call_remote_component( + self.processor_id, + 'filter', + filter_func=filter_func, + dataset_meta=dataset_meta, + init_args=init_args, + **kwargs) def add_dataset(self, dataset_meta: DatasetMeta, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'add_dataset', - **{'dataset_meta': dataset_meta}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - - - def mix_dataset(self, interleave = True): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'mix_dataset', - **{'interleave': interleave}, - } - ) - response.raise_for_status() - return response.json()["result"] - - - def save_as(self, output_path: str, format: Optional[str] = None, batch_size: int = 1000, mode: str = 'immediate', **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'save_as', - **{'output_path': output_path, 'format': format, 'batch_size': batch_size, 'mode': mode}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'add_dataset', dataset_meta=dataset_meta, **kwargs) + + def mix_dataset(self, interleave=True): + return call_remote_component(self.processor_id, 'mix_dataset', interleave=interleave) + + def save_as(self, + output_path: str, + format: Optional[str] = None, + batch_size: int = 1000, + mode: str = 'immediate', + **kwargs): + return call_remote_component( + self.processor_id, + 'save_as', + output_path=output_path, + format=format, + batch_size=batch_size, + mode=mode, + **kwargs) def flush_save(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'flush_save', - **{}, - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'flush_save') def __getitem__(self, idx): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__getitem__', - **{'idx': idx}, - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, '__getitem__', idx=idx) def __len__(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__len__', - **{}, - } - ) - response.raise_for_status() - return response.json()["result"] - \ No newline at end of file + return call_remote_component(self.processor_id, '__len__') diff --git a/src/twinkle_client/dataset/iterable_dataset.py b/src/twinkle_client/dataset/iterable_dataset.py index 0646ea33..0d7b5665 100644 --- a/src/twinkle_client/dataset/iterable_dataset.py +++ b/src/twinkle_client/dataset/iterable_dataset.py @@ -1,88 +1,28 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +from torch.utils.data import IterableDataset as TorchIterableDataset -from twinkle_client.http import http_post -from twinkle.dataset import Dataset from twinkle.dataset import DatasetMeta -from torch.utils.data import IterableDataset +from twinkle_client.common.component_rpc import call_remote_component, create_remote_component -class IterableDataset(IterableDataset): + +class IterableDataset(TorchIterableDataset): """Client wrapper for IterableDataset that calls server HTTP endpoints.""" def __init__(self, dataset_meta: DatasetMeta = None, **kwargs): - from twinkle_client.http import get_base_url - - self.server_url = f'{get_base_url()}/processor/twinkle' - response = http_post( - url=f'{self.server_url}/create', - json_data={ - 'processor_type': 'dataset', - 'class_type': 'IterableDataset', - **{'dataset_meta': dataset_meta}, **kwargs - } - ) - response.raise_for_status() - self.processor_id = response.json()['processor_id'] + self.processor_id = create_remote_component('dataset', 'IterableDataset', dataset_meta=dataset_meta, **kwargs) - def add_dataset(self, dataset_meta: DatasetMeta, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'add_dataset', - **{'dataset_meta': dataset_meta}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'add_dataset', dataset_meta=dataset_meta, **kwargs) def __len__(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__len__', - **{}, - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, '__len__') def __getitem__(self, idx): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__getitem__', - **{'idx': idx}, - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, '__getitem__', idx=idx) def __iter__(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__iter__', - **{}, - } - ) - response.raise_for_status() + call_remote_component(self.processor_id, '__iter__') return self - + def __next__(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__next__', - } - ) - response.raise_for_status() - return response.json()["result"] - \ No newline at end of file + return call_remote_component(self.processor_id, '__next__') diff --git a/src/twinkle_client/dataset/iterable_packing_dataset.py b/src/twinkle_client/dataset/iterable_packing_dataset.py index f4b6b5d1..012b5643 100644 --- a/src/twinkle_client/dataset/iterable_packing_dataset.py +++ b/src/twinkle_client/dataset/iterable_packing_dataset.py @@ -1,77 +1,39 @@ - +# Copyright (c) ModelScope Contributors. All rights reserved. +from torch.utils.data import IterableDataset from typing import Type, Union -from twinkle_client.http import http_post -from twinkle.dataset import Dataset + from twinkle.dataset import DatasetMeta from twinkle.template import Template -from torch.utils.data import IterableDataset +from twinkle_client.common.component_rpc import call_remote_component, create_remote_component + class IterablePackingDataset(IterableDataset): """Client wrapper for IterablePackingDataset that calls server HTTP endpoints.""" - def __init__(self, dataset_meta: DatasetMeta = None, packing_interval: int = 128, packing_num_proc: int = 1, cyclic: bool = False, **kwargs): - from twinkle_client.http import get_base_url - - self.server_url = f'{get_base_url()}/processor/twinkle' - response = http_post( - url=f'{self.server_url}/create', - json_data={ - 'processor_type': 'dataset', - 'class_type': 'IterablePackingDataset', - **{'dataset_meta': dataset_meta, 'packing_interval': packing_interval, 'packing_num_proc': packing_num_proc, 'cyclic': cyclic}, **kwargs - } - ) - response.raise_for_status() - self.processor_id = response.json()['processor_id'] + def __init__(self, + dataset_meta: DatasetMeta = None, + packing_interval: int = 128, + packing_num_proc: int = 1, + cyclic: bool = False, + **kwargs): + self.processor_id = create_remote_component( + 'dataset', + 'IterablePackingDataset', + dataset_meta=dataset_meta, + packing_interval=packing_interval, + packing_num_proc=packing_num_proc, + cyclic=cyclic, + **kwargs) - def set_template(self, template_cls: Union[Type[Template], str, Template], **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'set_template', - **{'template_cls': template_cls}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'set_template', template_cls=template_cls, **kwargs) def pack_dataset(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'pack_dataset', - **{}, - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'pack_dataset') def __iter__(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__iter__', - **{}, - } - ) - response.raise_for_status() + call_remote_component(self.processor_id, '__iter__') return self - + def __next__(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__next__', - } - ) - response.raise_for_status() - return response.json()["result"] - \ No newline at end of file + return call_remote_component(self.processor_id, '__next__') diff --git a/src/twinkle_client/dataset/lazy_dataset.py b/src/twinkle_client/dataset/lazy_dataset.py index 7bf49c70..b1af790a 100644 --- a/src/twinkle_client/dataset/lazy_dataset.py +++ b/src/twinkle_client/dataset/lazy_dataset.py @@ -1,137 +1,59 @@ - +# Copyright (c) ModelScope Contributors. All rights reserved. from typing import Any, Callable, Dict, Optional, Type, Union -from twinkle_client.http import http_post -from twinkle.dataset import Dataset + from twinkle.dataset import DatasetMeta -from twinkle.preprocessor import DataFilter -from twinkle.preprocessor import Preprocessor +from twinkle.preprocessor import DataFilter, Preprocessor +from twinkle_client.common.component_rpc import call_remote_component, create_remote_component from .base import Dataset + class LazyDataset(Dataset): """Client wrapper for LazyDataset that calls server HTTP endpoints.""" def __init__(self, dataset_meta: DatasetMeta = None, **kwargs): - from twinkle_client.http import get_base_url - - self.server_url = f'{get_base_url()}/processor/twinkle' - response = http_post( - url=f'{self.server_url}/create', - json_data={ - 'processor_type': 'dataset', - 'class_type': 'LazyDataset', - **{'dataset_meta': dataset_meta}, **kwargs - } - ) - response.raise_for_status() - self.processor_id = response.json()['processor_id'] - - - def map(self, preprocess_func: Union[Preprocessor, Callable, str, Type[Preprocessor]], dataset_meta: DatasetMeta = None, init_args: Dict[str, Any] = None, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'map', - **{'preprocess_func': preprocess_func, 'dataset_meta': dataset_meta, 'init_args': init_args}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - - - def filter(self, filter_func: Union[Callable, str, Type[DataFilter], DataFilter], dataset_meta: DatasetMeta = None, init_args: Dict[str, Any] = None, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'filter', - **{'filter_func': filter_func, 'dataset_meta': dataset_meta, 'init_args': init_args}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - + self.processor_id = create_remote_component('dataset', 'LazyDataset', dataset_meta=dataset_meta, **kwargs) + + def map(self, + preprocess_func: Union[Preprocessor, Callable, str, Type[Preprocessor]], + dataset_meta: DatasetMeta = None, + init_args: Dict[str, Any] = None, + **kwargs): + return call_remote_component( + self.processor_id, + 'map', + preprocess_func=preprocess_func, + dataset_meta=dataset_meta, + init_args=init_args, + **kwargs) + + def filter(self, + filter_func: Union[Callable, str, Type[DataFilter], DataFilter], + dataset_meta: DatasetMeta = None, + init_args: Dict[str, Any] = None, + **kwargs): + return call_remote_component( + self.processor_id, + 'filter', + filter_func=filter_func, + dataset_meta=dataset_meta, + init_args=init_args, + **kwargs) def add_dataset(self, dataset_meta: DatasetMeta, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'add_dataset', - **{'dataset_meta': dataset_meta}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'add_dataset', dataset_meta=dataset_meta, **kwargs) - def mix_dataset(self, interleave = True): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'mix_dataset', - **{'interleave': interleave}, - } - ) - response.raise_for_status() - return response.json()["result"] - + def mix_dataset(self, interleave=True): + return call_remote_component(self.processor_id, 'mix_dataset', interleave=interleave) def encode(self, add_generation_prompt: bool = False, timeout: Optional[int] = 600, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'encode', - **{'add_generation_prompt': add_generation_prompt}, - **kwargs - }, - timeout=timeout - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component( + self.processor_id, 'encode', timeout, add_generation_prompt=add_generation_prompt, **kwargs) def check(self, **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'check', - **{}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'check', **kwargs) def __getitem__(self, idx): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__getitem__', - **{'idx': idx}, - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, '__getitem__', idx=idx) def __len__(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__len__', - **{}, - } - ) - response.raise_for_status() - return response.json()["result"] - \ No newline at end of file + return call_remote_component(self.processor_id, '__len__') diff --git a/src/twinkle_client/dataset/packing_dataset.py b/src/twinkle_client/dataset/packing_dataset.py index 85577767..cbf03d48 100644 --- a/src/twinkle_client/dataset/packing_dataset.py +++ b/src/twinkle_client/dataset/packing_dataset.py @@ -1,63 +1,21 @@ - -from twinkle_client.http import http_post -from twinkle.dataset import Dataset +# Copyright (c) ModelScope Contributors. All rights reserved. from twinkle.dataset import DatasetMeta +from twinkle_client.common.component_rpc import call_remote_component, create_remote_component from .base import Dataset + class PackingDataset(Dataset): """Client wrapper for PackingDataset that calls server HTTP endpoints.""" def __init__(self, dataset_meta: DatasetMeta = None, packing_num_proc: int = 1, **kwargs): - from twinkle_client.http import get_base_url - - self.server_url = f'{get_base_url()}/processor/twinkle' - response = http_post( - url=f'{self.server_url}/create', - json_data={ - 'processor_type': 'dataset', - 'class_type': 'PackingDataset', - **{'dataset_meta': dataset_meta, 'packing_num_proc': packing_num_proc}, **kwargs - } - ) - response.raise_for_status() - self.processor_id = response.json()['processor_id'] + self.processor_id = create_remote_component( + 'dataset', 'PackingDataset', dataset_meta=dataset_meta, packing_num_proc=packing_num_proc, **kwargs) - def pack_dataset(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': 'pack_dataset', - **{}, - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, 'pack_dataset') def __getitem__(self, index): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__getitem__', - **{'index': index}, - } - ) - response.raise_for_status() - return response.json()["result"] - + return call_remote_component(self.processor_id, '__getitem__', index=index) def __len__(self): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__len__', - **{}, - } - ) - response.raise_for_status() - return response.json()["result"] - \ No newline at end of file + return call_remote_component(self.processor_id, '__len__') diff --git a/src/twinkle_client/exceptions.py b/src/twinkle_client/exceptions.py index 86bdc3d2..de91bbd9 100644 --- a/src/twinkle_client/exceptions.py +++ b/src/twinkle_client/exceptions.py @@ -10,6 +10,8 @@ :class:`TaskWaitTimeoutError`, :class:`TaskRecordLostError`) do NOT inherit ``requests.HTTPError``: a task that reaches a ``failed`` terminal state is delivered over HTTP 200, so it is not an HTTP-level error. +- Request-construction failures (:class:`TwinkleClientValidationError`) happen before + any HTTP call is made. """ from __future__ import annotations @@ -18,6 +20,19 @@ import requests +class TwinkleClientValidationError(ValueError): + """A caller argument could not be placed in the request model, in-process. + + Distinct from ``pydantic.ValidationError``, which reports a *field* that failed + validation. This one is raised **before** the model is constructed, when an + argument has no field to go into at all -- so it cannot be expressed as a field + error. Either way no HTTP request is sent. + + A ``ValueError`` subclass so callers already catching ``ValueError`` around request + construction keep working. + """ + + class TwinkleHTTPError(requests.HTTPError): """An HTTP 4xx/5xx (other than 410) from a twinkle endpoint. diff --git a/src/twinkle_client/http/__init__.py b/src/twinkle_client/http/__init__.py index e36ce1e2..068cfa51 100644 --- a/src/twinkle_client/http/__init__.py +++ b/src/twinkle_client/http/__init__.py @@ -1,10 +1,11 @@ -from .http_utils import http_delete, http_get, http_post +from .http_utils import http_delete, http_get, http_post, http_post_model from .utils import (TWINKLE_SERVER_TOKEN, TWINKLE_SERVER_URL, get_api_key, get_base_url, get_request_id, get_session_id, set_api_key, set_base_url, set_request_id, set_session_id) __all__ = [ 'http_get', 'http_post', + 'http_post_model', 'http_delete', 'TWINKLE_SERVER_URL', 'TWINKLE_SERVER_TOKEN', diff --git a/src/twinkle_client/http/http_utils.py b/src/twinkle_client/http/http_utils.py index 3fc71c05..a96a52fd 100644 --- a/src/twinkle_client/http/http_utils.py +++ b/src/twinkle_client/http/http_utils.py @@ -169,6 +169,25 @@ def http_post( return _handle_response(response) +def http_post_model( + url: str, + body: Any, + additional_headers: Optional[Dict[str, str]] = None, + timeout: Optional[int] = _HTTP_TIMEOUT, +) -> requests.Response: + """POST a request model as its own JSON serialization. + + One pydantic-core pass produces the bytes, instead of ``model_dump`` followed by + ``_serialize_params``' Python-level walk and then ``json.dumps`` -- three traversals + of a body whose ``inputs`` can hold millions of elements. The model has already + validated everything, so nothing here needs to inspect the payload again. + """ + from twinkle_client._request_builder import request_json + headers = _build_headers({'content-type': 'application/json', **(additional_headers or {})}) + response = requests.post(url, headers=headers, data=request_json(body), timeout=timeout) + return _handle_response(response) + + def http_delete( url: Optional[str] = None, params: Optional[Dict[str, Any]] = {}, diff --git a/src/twinkle_client/model/multi_lora_transformers.py b/src/twinkle_client/model/multi_lora_transformers.py index 0416acac..a32029a9 100644 --- a/src/twinkle_client/model/multi_lora_transformers.py +++ b/src/twinkle_client/model/multi_lora_transformers.py @@ -1,33 +1,25 @@ -from typing import Any, Dict, Optional import itertools import logging import threading from pathlib import Path -from twinkle_client.http import http_post -from twinkle_client.common.json_utils import json_safe +from typing import Any, Dict, Optional + +from twinkle_client._request_builder import build_request +from twinkle_client.http import http_post, http_post_model +from twinkle_client.types import model as model_types from twinkle_client.types.component import DataRef -from twinkle_client.types.model import ( - CalculateLossResponse, - CalculateMetricResponse, - ClipGradNormResponse, - ForwardBackwardResponse, - ForwardResponse, - GetTrainConfigsResponse, - SaveResponse, - TrainingProgressResponse, -) logger = logging.getLogger('twinkle_client') -def _data_ref_payload(inputs: DataRef | list[DataRef]) -> dict[str, Any]: +def _data_refs(inputs: DataRef | list[DataRef]) -> list[dict[str, Any]]: """Encode one or more opaque references for a DataPlane model endpoint.""" refs = [inputs] if isinstance(inputs, DataRef) else list(inputs) if not refs: raise ValueError('at least one DataRef is required') if not all(isinstance(item, DataRef) for item in refs): raise TypeError('data-plane model inputs must contain only DataRef values') - return {'input_refs': [item.model_dump() for item in refs]} + return [item.model_dump() for item in refs] class MultiLoraTransformersModel: @@ -35,6 +27,12 @@ class MultiLoraTransformersModel: This client manages adapters and sends training/inference requests to the model server. The server-side session (managed by TwinkleClient) keeps the model alive. + + Every method builds its endpoint's request model rather than a dict, so a + misspelled or wrongly-typed argument fails here -- in the caller's own stack trace, + with no request sent. Arguments that are not declared fields (loss inputs, plugin + constructor arguments) are routed into that model's passthrough region, so public + signatures stay ``**kwargs`` and callers are unchanged. """ def __init__(self, model_id: str, **kwargs): @@ -58,6 +56,16 @@ def __init__(self, model_id: str, **kwargs): ) response.raise_for_status() + # ------------------------------------------------------------------ # + # Request plumbing + # ------------------------------------------------------------------ # + + def _submit(self, endpoint: str, model_cls, response_cls, **values): + """Build, send, and resolve one twinkle-native request.""" + body = build_request(model_cls, **values) + response = http_post_model(f'{self.server_url}/{endpoint}', body) + return self._await_task(response, response_cls) + @staticmethod def _await_task(response, model_cls): """Resolve a Submit_Endpoint response through the Client_Future_Layer. @@ -74,6 +82,10 @@ def _next_seq_id(self) -> int: with self._seq_lock: return next(self._seq_counter) + # ------------------------------------------------------------------ # + # Adapter lifecycle + # ------------------------------------------------------------------ # + def add_adapter_to_model(self, adapter_name: str, config: Optional[Dict[str, Any]] = None, **kwargs) -> None: """Add a new adapter to the model. @@ -81,42 +93,92 @@ def add_adapter_to_model(self, adapter_name: str, config: Optional[Dict[str, Any LoRA-mode deployment. Pass ``config=None`` for full-parameter training against a ``train_mode: full`` deployment. """ - save_dir = kwargs.get('save_dir') + save_dir = kwargs.pop('save_dir', None) if save_dir: - kwargs['save_dir'] = Path(save_dir).expanduser().resolve().as_posix() - response = http_post( - url=f'{self.server_url}/add_adapter_to_model', - json_data={'adapter_name': adapter_name, 'config': config, **kwargs} - ) - self._await_task(response, None) + save_dir = Path(save_dir).expanduser().resolve().as_posix() + self._submit( + 'add_adapter_to_model', + model_types.AddAdapterRequest, + None, + adapter_name=adapter_name, + config=config, + save_dir=save_dir, + **kwargs) self.adapter_name = adapter_name def remove_adapter(self, adapter_name: str | None = None) -> None: """Release one client-owned adapter from the training component.""" name = adapter_name or self.adapter_name - response = http_post( - url=f'{self.server_url}/remove_adapter', - json_data={'adapter_name': name}, - ) - self._await_task(response, None) + self._submit('remove_adapter', model_types.AdapterRequest, None, adapter_name=name) if name == self.adapter_name: self.adapter_name = None - def forward(self, inputs: Any, **kwargs) -> ForwardResponse: - """Execute forward pass on inline model inputs.""" - response = http_post( - url=f'{self.server_url}/forward', - json_data={'inputs': inputs, 'adapter_name': self.adapter_name, **kwargs}, - ) - return self._await_task(response, ForwardResponse) + # ------------------------------------------------------------------ # + # Inline forward family + # ------------------------------------------------------------------ # - def forward_only(self, inputs: Any, **kwargs) -> ForwardResponse: + def forward(self, inputs: Any, **kwargs) -> model_types.ForwardResponse: + """Execute forward pass on inline model inputs.""" + return self._submit( + 'forward', + model_types.ForwardRequest, + model_types.ForwardResponse, + inputs=inputs, + adapter_name=self.adapter_name, + **kwargs) + + def forward_only(self, inputs: Any, **kwargs) -> model_types.ForwardResponse: """Execute forward pass without gradient computation on inline inputs.""" - response = http_post( - url=f'{self.server_url}/forward_only', - json_data={'inputs': inputs, 'adapter_name': self.adapter_name, **kwargs}, - ) - return self._await_task(response, ForwardResponse) + return self._submit( + 'forward_only', + model_types.ForwardOnlyRequest, + model_types.ForwardResponse, + inputs=inputs, + adapter_name=self.adapter_name, + **kwargs) + + def forward_backward(self, inputs: Any, **kwargs) -> model_types.ForwardBackwardResponse: + """Execute combined forward and backward pass on inline inputs.""" + return self._submit( + 'forward_backward', + model_types.ForwardBackwardTaskRequest, + model_types.ForwardBackwardResponse, + inputs=inputs, + adapter_name=self.adapter_name, + seq_id=self._next_seq_id(), + **kwargs) + + def calculate_loss(self, **kwargs) -> model_types.CalculateLossResponse: + """Calculate loss from model outputs.""" + return self._submit( + 'calculate_loss', + model_types.AdapterRequest, + model_types.CalculateLossResponse, + adapter_name=self.adapter_name, + **kwargs) + + def get_train_configs(self, **kwargs) -> model_types.GetTrainConfigsResponse: + """Get training configs.""" + return self._submit( + 'get_train_configs', + model_types.AdapterRequest, + model_types.GetTrainConfigsResponse, + adapter_name=self.adapter_name, + **kwargs) + + def backward(self, **kwargs) -> None: + """Execute backward pass.""" + self._submit( + 'backward', + model_types.AdapterRequest, + None, + adapter_name=self.adapter_name, + seq_id=self._next_seq_id(), + **kwargs) + + # ------------------------------------------------------------------ # + # Data-plane forward family + # ------------------------------------------------------------------ # def forward_from_data_plane( self, @@ -125,19 +187,17 @@ def forward_from_data_plane( input_field: str | None = None, kwarg_fields: dict[str, str] | None = None, **kwargs, - ) -> ForwardResponse: + ) -> model_types.ForwardResponse: """Execute forward using rows referenced from the server DataPlane.""" - response = http_post( - url=f'{self.server_url}/forward_from_data_plane', - json_data={ - **_data_ref_payload(inputs), - 'adapter_name': self.adapter_name, - 'input_field': input_field, - 'kwarg_fields': kwarg_fields or {}, - **json_safe(kwargs), - }, - ) - return self._await_task(response, ForwardResponse) + return self._submit( + 'forward_from_data_plane', + model_types.DataPlaneForwardRequest, + model_types.ForwardResponse, + input_refs=_data_refs(inputs), + adapter_name=self.adapter_name, + input_field=input_field, + kwarg_fields=kwarg_fields or {}, + **kwargs) def forward_only_from_data_plane( self, @@ -148,58 +208,23 @@ def forward_only_from_data_plane( output_ref: DataRef | None = None, output_fields: dict[str, str] | None = None, **kwargs, - ) -> ForwardResponse | DataRef: + ) -> model_types.ForwardResponse | DataRef: """Execute forward-only using DataPlane rows and optionally append outputs.""" - body = { - **_data_ref_payload(inputs), - 'adapter_name': self.adapter_name, - 'input_field': input_field, - 'kwarg_fields': kwarg_fields or {}, - 'output_ref': output_ref.model_dump() if output_ref is not None else None, - 'output_fields': output_fields or {}, - **json_safe(kwargs), - } - response = http_post( - url=f'{self.server_url}/forward_only_from_data_plane', - json_data=body, - ) - result = self._await_task(response, ForwardResponse) + result = self._submit( + 'forward_only_from_data_plane', + model_types.DataPlaneForwardOnlyRequest, + model_types.ForwardResponse, + input_refs=_data_refs(inputs), + adapter_name=self.adapter_name, + input_field=input_field, + kwarg_fields=kwarg_fields or {}, + output_ref=output_ref.model_dump() if output_ref is not None else None, + output_fields=output_fields or {}, + **kwargs) if output_ref is not None: return DataRef(**result.result) return result - def calculate_loss(self, **kwargs) -> CalculateLossResponse: - """Calculate loss from model outputs.""" - response = http_post( - url=f'{self.server_url}/calculate_loss', - json_data={'adapter_name': self.adapter_name, **kwargs} - ) - return self._await_task(response, CalculateLossResponse) - - def get_train_configs(self, **kwargs) -> GetTrainConfigsResponse: - """Get training configs.""" - response = http_post( - url=f'{self.server_url}/get_train_configs', - json_data={'adapter_name': self.adapter_name, **kwargs} - ) - return self._await_task(response, GetTrainConfigsResponse) - - def backward(self, **kwargs) -> None: - """Execute backward pass.""" - response = http_post( - url=f'{self.server_url}/backward', - json_data={'adapter_name': self.adapter_name, 'seq_id': self._next_seq_id(), **kwargs} - ) - self._await_task(response, None) - - def forward_backward(self, inputs: Any, **kwargs) -> ForwardBackwardResponse: - """Execute combined forward and backward pass on inline inputs.""" - response = http_post( - url=f'{self.server_url}/forward_backward', - json_data={'inputs': inputs, 'adapter_name': self.adapter_name, 'seq_id': self._next_seq_id(), **kwargs}, - ) - return self._await_task(response, ForwardBackwardResponse) - def forward_backward_from_data_plane( self, inputs: DataRef | list[DataRef], @@ -207,169 +232,188 @@ def forward_backward_from_data_plane( input_field: str | None = None, kwarg_fields: dict[str, str] | None = None, **kwargs, - ) -> ForwardBackwardResponse: + ) -> model_types.ForwardBackwardResponse: """Execute forward/backward using rows referenced from the server DataPlane.""" - response = http_post( - url=f'{self.server_url}/forward_backward_from_data_plane', - json_data={ - **_data_ref_payload(inputs), - 'adapter_name': self.adapter_name, - 'input_field': input_field, - 'kwarg_fields': kwarg_fields or {}, - 'seq_id': self._next_seq_id(), - **json_safe(kwargs), - }, - ) - return self._await_task(response, ForwardBackwardResponse) + return self._submit( + 'forward_backward_from_data_plane', + model_types.DataPlaneForwardRequest, + model_types.ForwardBackwardResponse, + input_refs=_data_refs(inputs), + adapter_name=self.adapter_name, + input_field=input_field, + kwarg_fields=kwarg_fields or {}, + seq_id=self._next_seq_id(), + **kwargs) + + # ------------------------------------------------------------------ # + # Optimizer / scheduler steps + # ------------------------------------------------------------------ # def step(self, **kwargs) -> None: """Execute optimizer step.""" - response = http_post( - url=f'{self.server_url}/step', - json_data={'adapter_name': self.adapter_name, 'seq_id': self._next_seq_id(), **kwargs} - ) - self._await_task(response, None) + self._submit( + 'step', + model_types.StepRequest, + None, + adapter_name=self.adapter_name, + seq_id=self._next_seq_id(), + **kwargs) def zero_grad(self, **kwargs) -> None: """Zero out gradients.""" - response = http_post( - url=f'{self.server_url}/zero_grad', - json_data={'adapter_name': self.adapter_name, **kwargs} - ) - self._await_task(response, None) + self._submit('zero_grad', model_types.AdapterRequest, None, adapter_name=self.adapter_name, **kwargs) def lr_step(self, **kwargs) -> None: """Execute learning rate scheduler step.""" - response = http_post( - url=f'{self.server_url}/lr_step', - json_data={'adapter_name': self.adapter_name, 'seq_id': self._next_seq_id(), **kwargs} - ) - self._await_task(response, None) - - def clip_grad_norm(self, max_grad_norm: float = 1.0, norm_type: int = 2, **kwargs) -> ClipGradNormResponse: + self._submit( + 'lr_step', + model_types.LrStepRequest, + None, + adapter_name=self.adapter_name, + seq_id=self._next_seq_id(), + **kwargs) + + def clip_grad_norm(self, + max_grad_norm: float = 1.0, + norm_type: int = 2, + **kwargs) -> model_types.ClipGradNormResponse: """Clip gradient norm.""" - response = http_post( - url=f'{self.server_url}/clip_grad_norm', - json_data={ - 'max_grad_norm': max_grad_norm, - 'norm_type': norm_type, - 'adapter_name': self.adapter_name, - **kwargs - } - ) - return self._await_task(response, ClipGradNormResponse) + return self._submit( + 'clip_grad_norm', + model_types.ClipGradNormRequest, + model_types.ClipGradNormResponse, + adapter_name=self.adapter_name, + max_grad_norm=max_grad_norm, + norm_type=norm_type, + **kwargs) def clip_grad_and_step(self, max_grad_norm: float = 1.0, norm_type: int = 2, **kwargs) -> None: """Clip gradient norm and execute optimizer step in one call.""" - response = http_post( - url=f'{self.server_url}/clip_grad_and_step', - json_data={ - 'max_grad_norm': max_grad_norm, - 'norm_type': norm_type, - 'adapter_name': self.adapter_name, - 'seq_id': self._next_seq_id(), - **kwargs - } - ) - self._await_task(response, None) + self._submit( + 'clip_grad_and_step', + model_types.ClipGradAndStepRequest, + None, + adapter_name=self.adapter_name, + max_grad_norm=max_grad_norm, + norm_type=norm_type, + seq_id=self._next_seq_id(), + **kwargs) + + # ------------------------------------------------------------------ # + # Plugin setters + # ------------------------------------------------------------------ # def set_loss(self, loss_cls: str, **kwargs) -> None: """Set the loss function.""" - response = http_post( - url=f'{self.server_url}/set_loss', - json_data={'loss_cls': loss_cls, 'adapter_name': self.adapter_name, **kwargs} - ) - self._await_task(response, None) + self._submit( + 'set_loss', model_types.SetLossRequest, None, loss_cls=loss_cls, adapter_name=self.adapter_name, **kwargs) def set_optimizer(self, optimizer_cls: str, **kwargs) -> None: """Set the optimizer.""" - response = http_post( - url=f'{self.server_url}/set_optimizer', - json_data={'optimizer_cls': optimizer_cls, 'adapter_name': self.adapter_name, **kwargs} - ) - self._await_task(response, None) + self._submit( + 'set_optimizer', + model_types.SetOptimizerRequest, + None, + optimizer_cls=optimizer_cls, + adapter_name=self.adapter_name, + **kwargs) def set_lr_scheduler(self, scheduler_cls: str, **kwargs) -> None: """Set the learning rate scheduler.""" - response = http_post( - url=f'{self.server_url}/set_lr_scheduler', - json_data={'scheduler_cls': scheduler_cls, 'adapter_name': self.adapter_name, **kwargs} - ) - self._await_task(response, None) - - def save(self, name: str, **kwargs) -> SaveResponse: - """Save model checkpoint.""" - response = http_post( - url=f'{self.server_url}/save', - json_data={'name': name, 'adapter_name': self.adapter_name, **kwargs} - ) - return self._await_task(response, SaveResponse) + self._submit( + 'set_lr_scheduler', + model_types.SetLrSchedulerRequest, + None, + scheduler_cls=scheduler_cls, + adapter_name=self.adapter_name, + **kwargs) - def load(self, name: str, **kwargs) -> None: - """Load model checkpoint.""" - response = http_post( - url=f'{self.server_url}/load', - json_data={'name': name, 'adapter_name': self.adapter_name, **kwargs} - ) - self._await_task(response, None) + def set_template(self, template_cls: str, **kwargs) -> None: + """Set the template for data processing. - def resume_from_checkpoint(self, name: str, *, resume_only_model: bool = False, **kwargs) -> Dict[str, Any]: - response = http_post( - url=f'{self.server_url}/resume_from_checkpoint', - json_data={'name': name, 'adapter_name': self.adapter_name, - 'resume_only_model': resume_only_model, **kwargs} - ) - return self._await_task(response, TrainingProgressResponse).result + ``model_id`` is not injected here: the backend always overrides it with its own + tokenizer id, so sending it made the request advertise a parameter that had no + effect. A caller that passes it explicitly still reaches the template + constructor through the passthrough region. + """ + self._submit( + 'set_template', + model_types.SetTemplateRequest, + None, + template_cls=template_cls, + adapter_name=self.adapter_name, + **kwargs) - def apply_patch(self, patch_cls: str, **kwargs) -> None: - """Apply a patch to the model.""" - response = http_post( - url=f'{self.server_url}/apply_patch', - json_data={'patch_cls': patch_cls, 'adapter_name': self.adapter_name, **kwargs} - ) - self._await_task(response, None) + def set_processor(self, processor_cls: str, **kwargs) -> None: + """Set the input processor.""" + self._submit( + 'set_processor', + model_types.SetProcessorRequest, + None, + processor_cls=processor_cls, + adapter_name=self.adapter_name, + **kwargs) def add_metric(self, metric_cls: str, is_training: Optional[bool] = None, **kwargs) -> None: """Add a metric to the model.""" - response = http_post( - url=f'{self.server_url}/add_metric', - json_data={ - 'metric_cls': metric_cls, - 'is_training': is_training, - 'adapter_name': self.adapter_name, - **kwargs - } - ) - self._await_task(response, None) + self._submit( + 'add_metric', + model_types.AddMetricRequest, + None, + metric_cls=metric_cls, + is_training=is_training, + adapter_name=self.adapter_name, + **kwargs) - def set_template(self, template_cls: str, **kwargs) -> None: - """Set the template for data processing.""" - response = http_post( - url=f'{self.server_url}/set_template', - json_data={ - 'template_cls': template_cls, - 'adapter_name': self.adapter_name, - 'model_id': self.model_id, - **kwargs - } - ) - self._await_task(response, None) + def apply_patch(self, patch_cls: str, **kwargs) -> None: + """Apply a patch to the model.""" + self._submit( + 'apply_patch', + model_types.ApplyPatchRequest, + None, + patch_cls=patch_cls, + adapter_name=self.adapter_name, + **kwargs) + + def calculate_metric(self, is_training: bool = True, **kwargs) -> model_types.CalculateMetricResponse: + """Calculate metrics from model outputs.""" + return self._submit( + 'calculate_metric', + model_types.CalculateMetricRequest, + model_types.CalculateMetricResponse, + is_training=is_training, + adapter_name=self.adapter_name, + **kwargs) + + # ------------------------------------------------------------------ # + # Checkpoint I/O + # ------------------------------------------------------------------ # + + def save(self, name: str, **kwargs) -> model_types.SaveResponse: + """Save model checkpoint.""" + return self._submit( + 'save', + model_types.SaveRequest, + model_types.SaveResponse, + name=name, + adapter_name=self.adapter_name, + **kwargs) - def set_processor(self, processor_cls: str, **kwargs) -> None: - """Set the input processor.""" - response = http_post( - url=f'{self.server_url}/set_processor', - json_data={'processor_cls': processor_cls, 'adapter_name': self.adapter_name, **kwargs} - ) - self._await_task(response, None) + def load(self, name: str, **kwargs) -> None: + """Load model checkpoint.""" + self._submit('load', model_types.LoadRequest, None, name=name, adapter_name=self.adapter_name, **kwargs) - def calculate_metric(self, is_training: bool = True, **kwargs) -> CalculateMetricResponse: - """Calculate metrics from model outputs.""" - response = http_post( - url=f'{self.server_url}/calculate_metric', - json_data={'is_training': is_training, 'adapter_name': self.adapter_name, **kwargs} - ) - return self._await_task(response, CalculateMetricResponse) + def resume_from_checkpoint(self, name: str, *, resume_only_model: bool = False, **kwargs) -> Dict[str, Any]: + """Resume weights (and optionally optimizer state) from a checkpoint.""" + progress = self._submit( + 'resume_from_checkpoint', + model_types.ResumeFromCheckpointRequest, + model_types.TrainingProgressResponse, + name=name, + adapter_name=self.adapter_name, + resume_only_model=resume_only_model, + **kwargs) + return progress.result def upload_to_hub( self, @@ -393,14 +437,12 @@ def upload_to_hub( poll_interval: Deprecated, has no effect. Pacing is now owned by the server-side long-poll of the Retrieve_Endpoint. """ - response = http_post( - url=f'{self.server_url}/upload_to_hub', - json_data={ - 'checkpoint_dir': checkpoint_dir, - 'hub_model_id': hub_model_id, - 'hub_token': hub_token, - } - ) - logger.info('[upload_to_hub] upload submitted, waiting for completion...') - self._await_task(response, None) + logger.info('[upload_to_hub] submitting upload, waiting for completion...') + self._submit( + 'upload_to_hub', + model_types.UploadToHubRequest, + None, + checkpoint_dir=checkpoint_dir, + hub_model_id=hub_model_id, + hub_token=hub_token) logger.info('[upload_to_hub] upload completed successfully.') diff --git a/src/twinkle_client/processor/base.py b/src/twinkle_client/processor/base.py index 37786527..45502209 100644 --- a/src/twinkle_client/processor/base.py +++ b/src/twinkle_client/processor/base.py @@ -1,38 +1,26 @@ - +# Copyright (c) ModelScope Contributors. All rights reserved. from typing import List, Literal, Optional, Union -from twinkle_client.http import http_post + from twinkle import DeviceMesh from twinkle.data_format import InputFeature +from twinkle_client.common.component_rpc import call_remote_component, create_remote_component + class InputProcessor(object): """Client wrapper for InputProcessor that calls server HTTP endpoints.""" - def __init__(self, device_mesh: Optional[DeviceMesh] = None, padding_free: bool = False, framework: Literal['transformers', 'megatron'] = 'transformers', **kwargs): - from twinkle_client.http import get_base_url - - self.server_url = f'{get_base_url()}/processor/twinkle' - response = http_post( - url=f'{self.server_url}/create', - json_data={ - 'processor_type': 'processor', - 'class_type': 'InputProcessor', - **{'device_mesh': device_mesh, 'padding_free': padding_free, 'framework': framework}, **kwargs - } - ) - response.raise_for_status() - self.processor_id = response.json()['processor_id'] + def __init__(self, + device_mesh: Optional[DeviceMesh] = None, + padding_free: bool = False, + framework: Literal['transformers', 'megatron'] = 'transformers', + **kwargs): + self.processor_id = create_remote_component( + 'processor', + 'InputProcessor', + device_mesh=device_mesh, + padding_free=padding_free, + framework=framework, + **kwargs) - def __call__(self, inputs: Union[InputFeature, List[InputFeature]], **kwargs): - response = http_post( - url=f'{self.server_url}/call', - json_data={ - 'processor_id': self.processor_id, - 'function': '__call__', - **{'inputs': inputs}, - **kwargs - } - ) - response.raise_for_status() - return response.json()["result"] - \ No newline at end of file + return call_remote_component(self.processor_id, '__call__', inputs=inputs, **kwargs) diff --git a/src/twinkle_client/sampler/vllm_sampler.py b/src/twinkle_client/sampler/vllm_sampler.py index 719b9d06..604032bb 100644 --- a/src/twinkle_client/sampler/vllm_sampler.py +++ b/src/twinkle_client/sampler/vllm_sampler.py @@ -1,13 +1,17 @@ import asyncio from dataclasses import asdict from typing import Any, Dict, List, Optional, Union -from twinkle_client.http import http_post -from twinkle_client.types.sampler import (AddAdapterResponse, SampleResponseModel, SampleResponseModelList, - SetTemplateResponse) + from peft import PeftConfig -from twinkle.data_format import Trajectory, InputFeature, SamplingParams + +from twinkle.data_format import InputFeature, SamplingParams, Trajectory +from twinkle_client._request_builder import build_request from twinkle_client.common.json_utils import json_safe -from twinkle_client.types.component import DataRef +from twinkle_client.http import http_post, http_post_model +from twinkle_client.types.component import DataPlaneSampleRequest, DataRef, UnloadAdapterPathsRequest +from twinkle_client.types.sampler import (SamplerAddAdapterRequest, SamplerAddAdapterResponse, SampleRequest, + SampleResponseModel, SampleResponseModelList, SamplerSetTemplateRequest, + SamplerSetTemplateResponse) # Intentionally does NOT subclass ``twinkle.sampler.base.Sampler``: importing @@ -61,17 +65,14 @@ def _await_task(response, model_cls): from twinkle_client._future import resolve_response return resolve_response(response, model_cls) - def add_adapter_to_sampler(self, adapter_name: str, config: PeftConfig, **kwargs) -> AddAdapterResponse: + def add_adapter_to_sampler(self, adapter_name: str, config: PeftConfig, **kwargs) -> SamplerAddAdapterResponse: """Add a new adapter to the sampler.""" if isinstance(config, PeftConfig): config = config.__dict__ - response = http_post( - url=f'{self.server_url}/add_adapter_to_sampler', - json_data={'adapter_name': adapter_name, 'config': config, **kwargs} - ) - response.raise_for_status() + body = build_request(SamplerAddAdapterRequest, adapter_name=adapter_name, config=config, **kwargs) + response = http_post_model(f'{self.server_url}/add_adapter_to_sampler', body) self.adapter_name = adapter_name - return AddAdapterResponse(**response.json()) + return SamplerAddAdapterResponse(**response.json()) def sample( self, @@ -93,25 +94,27 @@ def sample( Returns: SampleResponseModel with 'sequences' list, each containing tokens, logprobs, stop_reason. """ + response = http_post_model( + f'{self.server_url}/sample', + build_request( + SampleRequest, + inputs=_json_safe(inputs), + sampling_params=self._sampling_params(sampling_params, num_samples), + adapter_name=adapter_name, + adapter_uri=adapter_uri)) + return self._await_task(response, SampleResponseModelList).samples + + @staticmethod + def _sampling_params(sampling_params: Optional[Union[SamplingParams, Dict[str, Any]]], + num_samples: int) -> Dict[str, Any]: + """Normalise sampling parameters into the single dict the server builds from.""" if isinstance(sampling_params, SamplingParams): sampling_params = asdict(sampling_params) else: sampling_params = dict(sampling_params or {}) if num_samples != 1 and sampling_params.setdefault('num_samples', num_samples) != num_samples: raise ValueError('num_samples conflicts with sampling_params.num_samples') - json_data = { - 'inputs': _json_safe(inputs), - 'sampling_params': _json_safe(sampling_params), - 'adapter_name': adapter_name, - } - if adapter_uri is not None: - json_data['adapter_uri'] = adapter_uri - - response = http_post( - url=f'{self.server_url}/sample', - json_data=json_data - ) - return self._await_task(response, SampleResponseModelList).samples + return _json_safe(sampling_params) def sample_to_data_plane( self, @@ -125,20 +128,21 @@ def sample_to_data_plane( num_samples: int = 1, ) -> DataRef: """Generate complete prompt groups and keep their rows in the server DataPlane.""" - body = { - 'sampling_params': sampling_params, - 'adapter_name': adapter_name, - 'adapter_uri': adapter_uri, - 'policy_version': policy_version, - 'group_ids': group_ids, - 'num_samples': num_samples, - } - body['input_ref' if isinstance(inputs, DataRef) else 'inputs'] = ( - inputs.model_dump() if isinstance(inputs, DataRef) else _json_safe(inputs)) - response = http_post( - url=f'{self.server_url}/sample_to_data_plane', - json_data=json_safe(body), - ) + source = ({ + 'input_ref': inputs.model_dump() + } if isinstance(inputs, DataRef) else { + 'inputs': _json_safe(inputs) + }) + body = build_request( + DataPlaneSampleRequest, + sampling_params=_json_safe(sampling_params) if sampling_params else None, + adapter_name=adapter_name, + adapter_uri=adapter_uri, + policy_version=policy_version, + group_ids=group_ids, + num_samples=num_samples, + **source) + response = http_post_model(f'{self.server_url}/sample_to_data_plane', body) return self._await_task(response, DataRef) async def asample( @@ -184,25 +188,19 @@ async def asample_to_data_plane( def unload_adapter_paths(self, adapter_paths: list[str]) -> None: """Evict policy snapshots that are no longer referenced by this client.""" - response = http_post( - url=f'{self.server_url}/unload_adapter_paths', - json_data={'adapter_paths': adapter_paths}, - ) - response.raise_for_status() + http_post_model( + f'{self.server_url}/unload_adapter_paths', + build_request(UnloadAdapterPathsRequest, adapter_paths=adapter_paths)) - def set_template(self, template_cls: str, adapter_name: str = '', **kwargs) -> SetTemplateResponse: + def set_template(self, template_cls: str, adapter_name: str = '', **kwargs) -> SamplerSetTemplateResponse: """Set the template for encoding trajectories.""" - response = http_post( - url=f'{self.server_url}/set_template', - json_data={'template_cls': template_cls, 'adapter_name': adapter_name, **kwargs} - ) - response.raise_for_status() - return SetTemplateResponse(**response.json()) + body = build_request(SamplerSetTemplateRequest, template_cls=template_cls, adapter_name=adapter_name, **kwargs) + response = http_post_model(f'{self.server_url}/set_template', body) + return SamplerSetTemplateResponse(**response.json()) def apply_patch(self, patch_cls: str, **kwargs) -> None: """Apply a patch to the model.""" - response = http_post( - url=f'{self.server_url}/apply_patch', - json_data={'patch_cls': patch_cls, 'adapter_name': self.adapter_name, **kwargs} - ) - response.raise_for_status() + from twinkle_client.types.model import ApplyPatchRequest + body = build_request( + ApplyPatchRequest, patch_cls=patch_cls, adapter_name=self.adapter_name or '', **kwargs) + http_post_model(f'{self.server_url}/apply_patch', body) diff --git a/src/twinkle_client/types/__init__.py b/src/twinkle_client/types/__init__.py index 35cbfbaf..9270658a 100644 --- a/src/twinkle_client/types/__init__.py +++ b/src/twinkle_client/types/__init__.py @@ -12,19 +12,21 @@ CalculateMetricResponse, ClipGradAndStepRequest, ClipGradAndStepResponse, + ClipGradNormRequest, ClipGradNormResponse, CreateRequest, CreateResponse, DataPlaneForwardOnlyRequest, DataPlaneForwardRequest, ForwardBackwardResponse, + ForwardBackwardTaskRequest, ForwardOnlyRequest, ForwardRequest, ForwardResponse, - GetStateDictRequest, GetTrainConfigsResponse, LoadRequest, LoadResponse, + LrStepRequest, LrStepResponse, ModelResult, OkResponse, @@ -41,6 +43,7 @@ SetProcessorResponse, SetTemplateRequest, SetTemplateResponse, + StepRequest, StepResponse, TrainingProgressResponse, UploadToHubRequest, @@ -55,15 +58,15 @@ ProcessorHeartbeatResponse, ) from .sampler import ( - AddAdapterRequest as SamplerAddAdapterRequest, - AddAdapterResponse, - CreateResponse as SamplerCreateResponse, SampledSequenceModel, SampleRequest, SampleResponseModel, SampleResponseModelList, - SetTemplateRequest as SamplerSetTemplateRequest, - SetTemplateResponse as SamplerSetTemplateResponse, + SamplerAddAdapterRequest, + SamplerAddAdapterResponse, + SamplerCreateResponse, + SamplerSetTemplateRequest, + SamplerSetTemplateResponse, ) from .server import ( CheckpointPathResponse, @@ -73,7 +76,6 @@ HealthResponse, SupportedModel, WeightsInfoRequest, - WeightsInfoResponse as ServerWeightsInfoResponse, CapacityInfoResponse, ) from .session import CreateSessionRequest, CreateSessionResponse, SessionHeartbeatRequest, SessionHeartbeatResponse @@ -90,6 +92,30 @@ ) from .checkpoint import ResolvedLoadPath +from .base import ( + BACKEND_ONLY_KEY, + DataModel, + FieldRole, + ResponseModel, + StrictRequest, + backend_kwarg, + backend_only, + fields_with_role, + passthrough, + read_backend_only, + read_field_role, +) +from .data import ( + CORE_INPUT_KEYS, + VLM_TENSOR_FIELDS, + WireInputBatch, + WireInputFeature, + WireInputs, + WireMessage, + WireTrajectory, + declared_wire_keys, + export_batch, +) from .lifecycle import CancelRequest, CancelResponse, RetrieveFutureRequest, TaskEnvelope, TaskStatus, TERMINAL_STATUSES from .component import ( DataAppendRequest, diff --git a/src/twinkle_client/types/base.py b/src/twinkle_client/types/base.py index ff06b8f1..4737a098 100644 --- a/src/twinkle_client/types/base.py +++ b/src/twinkle_client/types/base.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Shared pydantic base classes and the naming rulings for the wire contract. +"""Shared pydantic base classes, field roles, and the naming rulings for the wire contract. This module is a public contract carrier imported across packages (Twinkle_Server reverse-imports ``twinkle_client.types``); it therefore intentionally carries **no** @@ -24,12 +24,30 @@ 4. A closed value set on a wire field is declared as ``Literal`` / enum, never a bare ``str`` (see ``QueueStateLiteral`` in ``errors.py``). -These three base classes are DEFINED here but NOT applied to any existing model by -this spec: applying ``extra='forbid'`` would immediately reject an old client's -request, which would break the zero-wire-change guarantee. +Field roles +----------- +Every declared request field has exactly one role, and the role -- not the field +name -- decides whether it reaches the backend: + +- ``Control`` (the default): consumed by the handler itself, or passed explicitly + as a named argument. ``inputs``, ``adapter_name``, ``seq_id`` and the data-plane + reference fields are control fields. Forwarding them again through + ``**backend_kwargs`` would either duplicate a keyword argument or leak a + protocol field into a backend signature. +- ``BackendKwarg``: a user-facing backend parameter. Forwarded when, and only + when, its value is not ``None``. +- ``Passthrough``: a declared dict whose *keys* are dynamic. Its contents are + flattened into the backend kwargs, and its keys are exempt from + ``extra='forbid'`` (that setting constrains the model's own field set, not the + inside of a declared dict). The keys are forwarded as given -- see + :func:`passthrough` for why they are not checked against the target's signature. + +The role lives in the field's ``json_schema_extra`` so that one declaration site +carries it -- there is deliberately no second per-endpoint parameter table. """ from __future__ import annotations +from enum import StrEnum from pydantic import BaseModel, ConfigDict, Field from pydantic.fields import FieldInfo from typing import Any, Optional @@ -50,41 +68,102 @@ class ResponseModel(BaseModel): class DataModel(BaseModel): """Data-plane models (InputFeature / Trajectory on the wire). - Same ConfigDict as ResponseModel, different reason -- which is why this is a - separate class and not an alias. ResponseModel's ``ignore`` exists so an old - client tolerates new response fields. DataModel's ``ignore`` exists so a - user's Preprocessor / Template may leave harmless extra keys (the original - columns left by ``dataset.map``, say) without the request being rejected. - - Do NOT "fix" this to inherit StrictRequest. Doing so rejects those extra keys - and breaks a large number of existing datasets. + ``extra='allow'``, not ``forbid`` and not ``ignore``, and the difference is + load-bearing. A user's ``Preprocessor`` / ``Template`` routinely leaves extra + columns on an entry (whatever ``dataset.map`` did not drop). Rejecting those + would break a large number of existing datasets -- but *ignoring* them is just + as wrong, because the entry is then re-exported to the backend without them, + silently dropping data the caller sent. ``allow`` keeps unknown keys on the + model so :meth:`export` can hand them back. + + Do NOT "fix" this to ``StrictRequest`` or to ``extra='ignore'``. Strictness on + the data plane belongs to the *declared* fields (the ones Twinkle_Core reads), + which carry strict types; it does not belong to the field set. """ - model_config = ConfigDict(frozen=True, extra='ignore') + model_config = ConfigDict(frozen=True, extra='allow') + +class FieldRole(StrEnum): + """How a declared request field relates to the backend call.""" -# Key under which backend-applicability metadata is stored in a field's -# ``json_schema_extra``. A single constant, helper and reader -- kept here with the -# base classes rather than in a module of their own (no isolation benefit). + Control = 'control' + BackendKwarg = 'backend_kwarg' + Passthrough = 'passthrough' + + +# Keys under which field metadata is stored in a field's ``json_schema_extra``. +FIELD_ROLE_KEY = 'twinkle_field_role' BACKEND_ONLY_KEY = 'twinkle_backend_only' +def _with_extra(field_kwargs: dict[str, Any], **extra: Any) -> FieldInfo: + merged = dict(field_kwargs.pop('json_schema_extra', None) or {}) + merged.update(extra) + return Field(json_schema_extra=merged, **field_kwargs) + + +def backend_kwarg(*backends: str, **field_kwargs: Any) -> FieldInfo: + """Declare a field as a backend keyword argument. + + With no ``backends`` the field applies to every backend. Naming one or more + restricts it: sending a non-``None`` value to a deployment running a different + backend is rejected before the task is enqueued. + + A restricted field MUST be ``Optional[...] = None``. Giving it a backend's + constant default would make it carry a non-``None`` value on every deployment, + so the "non-``None`` on the wrong backend" check would reject every request. + """ + return _with_extra(field_kwargs, **{ + FIELD_ROLE_KEY: FieldRole.BackendKwarg.value, + BACKEND_ONLY_KEY: tuple(backends) or None, + }) + + def backend_only(*backends: str, **field_kwargs: Any) -> FieldInfo: - """Mark a model field as applicable only to the given backend(s). + """A backend keyword argument restricted to the given backend(s). - Attaches the backend tuple to the field's ``json_schema_extra`` under - ``BACKEND_ONLY_KEY``; read it back with :func:`read_backend_only`. + Kept as a named entry point because "this parameter only exists on megatron" + is the property a reader looks for at the declaration site; it is + :func:`backend_kwarg` with a non-empty backend tuple, not a second mechanism. """ - extra = dict(field_kwargs.pop('json_schema_extra', None) or {}) - extra[BACKEND_ONLY_KEY] = tuple(backends) - return Field(json_schema_extra=extra, **field_kwargs) + if not backends: + raise ValueError('backend_only() requires at least one backend; use backend_kwarg() for an unrestricted field') + return backend_kwarg(*backends, **field_kwargs) -def read_backend_only(field_info: FieldInfo) -> Optional[tuple[str, ...]]: - """Return the backend tuple a field was tagged with, or ``None`` if untagged.""" +def passthrough(**field_kwargs: Any) -> FieldInfo: + """Declare a dict field whose keys are dynamic backend parameters. + + Its contents are forwarded to the backend as given. They are deliberately not + checked against the target's signature: a plugin routinely reads a real parameter + straight out of ``**kwargs`` (``InputProcessor`` does this with ``padding_side``), + and ``inspect.signature`` cannot see such a read -- so any such check rejects valid + requests. A misspelled plugin argument therefore still surfaces from the plugin. + """ + field_kwargs.setdefault('default_factory', dict) + return _with_extra(field_kwargs, **{FIELD_ROLE_KEY: FieldRole.Passthrough.value}) + + +def _read_extra(field_info: FieldInfo, key: str) -> Any: extra = getattr(field_info, 'json_schema_extra', None) if isinstance(extra, dict): - value = extra.get(BACKEND_ONLY_KEY) - if value is not None: - return tuple(value) + return extra.get(key) return None + + +def read_field_role(field_info: FieldInfo) -> FieldRole: + """The field's role; ``Control`` when undeclared.""" + value = _read_extra(field_info, FIELD_ROLE_KEY) + return FieldRole(value) if value is not None else FieldRole.Control + + +def read_backend_only(field_info: FieldInfo) -> Optional[tuple[str, ...]]: + """Return the backend tuple a field was restricted to, or ``None`` if unrestricted.""" + value = _read_extra(field_info, BACKEND_ONLY_KEY) + return tuple(value) if value else None + + +def fields_with_role(model_cls: type[BaseModel], role: FieldRole) -> dict[str, FieldInfo]: + """The model's declared fields carrying ``role``, in declaration order.""" + return {name: info for name, info in model_cls.model_fields.items() if read_field_role(info) is role} diff --git a/src/twinkle_client/types/component.py b/src/twinkle_client/types/component.py index d7e9ec2a..4c7762ee 100644 --- a/src/twinkle_client/types/component.py +++ b/src/twinkle_client/types/component.py @@ -2,13 +2,21 @@ """Protocol types for directly orchestrating asynchronous server components.""" from __future__ import annotations -from typing import Any +from pydantic import BaseModel, Field, JsonValue, model_validator +from typing import Any, Optional -from pydantic import BaseModel, Field, model_validator +from .base import ResponseModel, StrictRequest +from .data import WireInputBatch class DataRef(BaseModel): - """Opaque reference to rows stored in the server-side TransferQueue.""" + """Opaque reference to rows stored in the server-side TransferQueue. + + A value carried inside other bodies rather than a body of its own, and it is + round-tripped by the client, so it keeps the plain base. No wire schema is applied + to what it points at: the rows never travel in the request body, so the data-plane + constraints would be a category error here. + """ ref_id: str size: int @@ -17,53 +25,57 @@ class DataRef(BaseModel): num_tokens: int = 0 -class DataPutRequest(BaseModel): +class DataPutRequest(StrictRequest): rows: list[dict[str, Any]] kind: str = 'data' - tags: list[dict[str, Any]] | None = None + tags: Optional[list[dict[str, Any]]] = None -class DataGetRequest(BaseModel): +class DataGetRequest(StrictRequest): ref: DataRef - fields: list[str] | None = None + fields: Optional[list[str]] = None include_tags: bool = False -class DataAppendRequest(BaseModel): +class DataAppendRequest(StrictRequest): ref: DataRef rows: list[dict[str, Any]] - tags: list[dict[str, Any]] | None = None + tags: Optional[list[dict[str, Any]]] = None -class DataReleaseRequest(BaseModel): +class DataReleaseRequest(StrictRequest): ref: DataRef -class DataRowsResponse(BaseModel): +class DataRowsResponse(ResponseModel): rows: list[dict[str, Any]] tags: list[dict[str, Any]] = Field(default_factory=list) -class DataPlaneSampleRequest(BaseModel): - inputs: Any = None - input_ref: DataRef | None = None - sampling_params: dict[str, Any] | None = None +class DataPlaneSampleRequest(StrictRequest): + """Body of ``POST /twinkle/sample_to_data_plane``. + + Exactly one input source: inline entries (wire-validated) or a ``DataRef``. + """ + + inputs: Optional[WireInputBatch] = None + input_ref: Optional[DataRef] = None + sampling_params: Optional[dict[str, JsonValue]] = None adapter_name: str = '' - adapter_uri: str | None = None - policy_version: int | None = None - group_ids: list[str] | None = None - num_samples: int = 1 + adapter_uri: Optional[str] = None + policy_version: Optional[int] = None + group_ids: Optional[list[str]] = None + num_samples: int = Field(default=1, ge=1) @model_validator(mode='after') def validate_input(self) -> 'DataPlaneSampleRequest': if (self.inputs is None) == (self.input_ref is None): raise ValueError('exactly one of inputs and input_ref must be provided') if self.group_ids is not None and self.inputs is not None: - size = len(self.inputs) if isinstance(self.inputs, list) else 1 - if len(self.group_ids) != size: + if len(self.group_ids) != len(self.inputs): raise ValueError('group_ids must contain one value per sampler input') return self -class UnloadAdapterPathsRequest(BaseModel): +class UnloadAdapterPathsRequest(StrictRequest): adapter_paths: list[str] diff --git a/src/twinkle_client/types/data.py b/src/twinkle_client/types/data.py new file mode 100644 index 00000000..f7bda96b --- /dev/null +++ b/src/twinkle_client/types/data.py @@ -0,0 +1,216 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Wire schema for the inline ``inputs`` data plane. + +These models are the *declared* type of the ``inputs`` request field, so FastAPI +validates a batch during body parsing -- before the handler runs, before a future +record exists, and before anything reaches a GPU. The seam that hands data to the +backend (``twinkle.server.lifecycle.submit.to_backend_inputs``) therefore only +exports an already-valid object; it is not the first place a malformed batch is +noticed. + +Two asymmetries are deliberate: + +- **Strict on declared fields, open on the field set.** Every field Twinkle_Core + reads is declared with a strict type (``StrictInt`` leaves reject ``true`` and + ``1.0``), while unknown JSON-native keys are kept and re-exported: a user's + preprocessor may leave extra columns on an entry and dropping them would lose + data the caller sent. See :class:`~twinkle_client.types.base.DataModel`. +- **Shallowest-first unions.** Nesting depth encodes tensor rank here, so a rank + range needs a union. Declaring the deepest branch first is a large, silent + pessimisation: given a 2-D input the 3-D branch does not fail at element 0, it + descends into every row and records one error per element. Measured on pydantic + 2.13.4 with a 1024 x 8192 (8.4M element) 2-D ``input_ids``: + + List[List[int]] single type 0.141 s + Union[3D, 2D, 1D] deepest-first 5.368 s <-- 41x worse + Union[1D, 2D, 3D] shallowest-first 0.131 s + + Do NOT reorder these to deepest-first. ``test_wire_schema.py`` asserts the + declared depth sequence is strictly increasing, so the ordering is checked + structurally rather than by a flaky timing benchmark. + +Known technical debt: encoding tensor shape in JSON nesting depth is what forces +the unions and makes validation cost scale with element count. The target shape is +a flat ``{dtype, shape, data}`` tensor envelope (which is what ``tinker`` uses). +Migrating is a breaking wire change and is out of scope here; do not paper over it +with hand-written Python-level depth checks or extra union branches, which would +add to the debt rather than pay it down. +""" +from __future__ import annotations + +from collections.abc import Mapping +from pydantic import BeforeValidator, Field, StrictInt, model_validator +from typing import Annotated, Any, List, Literal, Optional, Union + +from twinkle_client.types.base import DataModel +from twinkle.data_format.encoding import ENCODED_INPUT_KEYS + +# --------------------------------------------------------------------------- # +# Leaf types. Shallowest-first, and ``StrictInt`` wherever the values come from a +# tensor's ``tolist()`` -- there, a bool or a float is an upstream defect. Lax +# ``int`` coercion would turn ``[true, false]`` into ``[1, 0]``. +# +# ``union_mode='left_to_right'`` makes the declared order load-bearing instead of +# leaving branch selection to pydantic's heuristics. +# --------------------------------------------------------------------------- # + +_LEFT_TO_RIGHT = Field(union_mode='left_to_right') + +Ints1to2 = Annotated[Union[List[StrictInt], List[List[StrictInt]]], _LEFT_TO_RIGHT] +Ints1to3 = Annotated[Union[List[StrictInt], List[List[StrictInt]], List[List[List[StrictInt]]]], _LEFT_TO_RIGHT] +Ints3 = List[List[List[StrictInt]]] + +_Number = Union[StrictInt, float] +Numbers1to2 = Annotated[Union[List[_Number], List[List[_Number]]], _LEFT_TO_RIGHT] +Numbers1to4 = Annotated[Union[List[_Number], List[List[_Number]], List[List[List[_Number]]], + List[List[List[List[_Number]]]]], _LEFT_TO_RIGHT] + +# Media references travel as strings on the wire (local path, ``http(s)://`` URL, or +# a ``data:`` base64 URI). ``PIL.Image`` / raw ``bytes`` / ``np.ndarray`` are valid in +# the in-process training path but are not JSON, so they are not declared here. +MediaList = List[str] + +# The VLM tensor fields batched by concatenation rather than padding. Declared here +# because this module must stay free of Twinkle_Core's heavyweight imports; a +# consistency test asserts this set equals ``InputProcessor.VLM_CONCAT_FIELDS``, so a +# future addition there fails loudly instead of being silently dropped on the wire. +VLM_TENSOR_FIELDS: frozenset[str] = frozenset({ + 'pixel_values', + 'image_grid_thw', + 'pixel_values_videos', + 'video_grid_thw', + 'input_features', + 'input_features_mask', + 'feature_attention_mask', + 'grid_thws', +}) + + +class WireMessage(DataModel): + """One conversation turn, as sent over HTTP.""" + + role: Optional[Literal['system', 'user', 'assistant', 'tool']] = None + type: Optional[str] = None + content: Optional[Union[str, List[dict[str, Any]]]] = None + tool_calls: Optional[List[dict[str, Any]]] = None + tool_call_id: Optional[str] = None + reasoning_content: Optional[str] = None + images: Optional[MediaList] = None + videos: Optional[MediaList] = None + audios: Optional[MediaList] = None + + +class WireInputFeature(DataModel): + """An already-encoded entry: token ids (or embeddings) plus aligned tensors.""" + + input_ids: Optional[Ints1to2] = None + input_embedding: Optional[Numbers1to2] = None + attention_mask: Optional[Ints1to2] = None + labels: Optional[Ints1to2] = None + completion_mask: Optional[Ints1to2] = None + # 1-D standard encoding, 2-D Qwen-VL mrope ``[3, T]``, 3-D megatron ``[3, 1, N]``. + position_ids: Optional[Ints1to3] = None + # Exactly ``[seq_len, num_layers, topk]``. + routed_experts: Optional[Ints3] = None + length: Optional[StrictInt] = None + + # VLM tensors: float values are normal here, so no strict-int leaves. + pixel_values: Optional[Numbers1to4] = None + image_grid_thw: Optional[Numbers1to4] = None + pixel_values_videos: Optional[Numbers1to4] = None + video_grid_thw: Optional[Numbers1to4] = None + input_features: Optional[Numbers1to4] = None + input_features_mask: Optional[Numbers1to4] = None + feature_attention_mask: Optional[Numbers1to4] = None + grid_thws: Optional[Numbers1to4] = None + + @model_validator(mode='after') + def require_encoded_key(self) -> 'WireInputFeature': + """At least one of the encoded-input keys must be present. + + Declared as a model validator rather than by making ``input_ids`` required: + an embedding-only batch is legitimately encoded, and this is the same rule + the backends apply (:data:`ENCODED_INPUT_KEYS`). + """ + if all(getattr(self, key, None) is None for key in ENCODED_INPUT_KEYS): + raise ValueError(f'an encoded entry requires one of {list(ENCODED_INPUT_KEYS)}') + return self + + +class WireTrajectory(DataModel): + """A not-yet-encoded entry: messages the server template will encode.""" + + messages: List[WireMessage] + images: Optional[MediaList] = None + videos: Optional[MediaList] = None + audios: Optional[MediaList] = None + tools: Optional[List[dict[str, Any]]] = None + # ``List[Tuple[str, str]]`` on the wire: the PyArrow-stable encoding of the + # user-data pairs attached by ``twinkle.data_format.attach_user_data``. + user_data: Optional[List[tuple[str, str]]] = None + + +# A batch is homogeneous: every entry is encoded, or none is. Expressed as a union of +# *lists* rather than a list of unions, so a mixed batch fails to match either branch +# instead of being silently accepted and blowing up inside the backend. Order is +# encoded-first, matching ``is_encoded``: a trajectory has neither encoded key, so it +# cannot satisfy ``WireInputFeature``'s validator. +WireInputs = Union[List[WireInputFeature], List[WireTrajectory]] + + +def _as_batch(value: Any) -> Any: + """Accept a single entry where a batch is expected. + + Callers have always been allowed to pass one mapping instead of a one-element + list; normalising here keeps that while letting the declared type stay a batch, + so downstream code has exactly one shape to handle. + """ + return [value] if isinstance(value, Mapping) else value + + +#: The declared type of an inline ``inputs`` request field. +WireInputBatch = Annotated[WireInputs, BeforeValidator(_as_batch)] + +# Every ``inputs`` key Twinkle_Core reads. Maintained by hand on purpose: an AST scan +# would have to follow aliases (``inp = inputs[i]`` then ``inp.get('x')``), i.e. do a +# local data-flow analysis, and its false negatives would *silently* disable the +# consistency check that is this schema's only safety net against a dropped field. +# When adding a read of a new ``inputs`` key, add it here. +CORE_INPUT_KEYS: frozenset[str] = frozenset({ + 'input_ids', + 'input_embedding', + 'attention_mask', + 'labels', + 'completion_mask', + 'position_ids', + 'routed_experts', + 'length', + 'messages', + 'images', + 'videos', + 'audios', + 'tools', + 'user_data', +}) | VLM_TENSOR_FIELDS + + +def declared_wire_keys() -> frozenset[str]: + """Union of the field names declared across the wire input models.""" + return frozenset(WireInputFeature.model_fields) | frozenset(WireTrajectory.model_fields) + + +def export(entry: Union[WireInputFeature, WireTrajectory]) -> dict[str, Any]: + """Render a validated entry as the plain dict the backend consumes. + + ``exclude_none=True`` is required, not cosmetic: Twinkle_Core branches on key + *presence* in many places (``is_encoded``, ``inputs.pop('labels', None)``, the + VLM concat fields), so emitting unset optionals as ``None`` would change + behaviour. Unknown keys the caller sent are preserved -- that is why + :class:`DataModel` uses ``extra='allow'``. + """ + return entry.model_dump(exclude_none=True) + + +def export_batch(entries: List[Any]) -> List[dict[str, Any]]: + """Export a validated batch, leaving already-plain entries untouched.""" + return [export(entry) if isinstance(entry, DataModel) else entry for entry in entries] diff --git a/src/twinkle_client/types/model.py b/src/twinkle_client/types/model.py index bb8b7db8..554b4f1e 100644 --- a/src/twinkle_client/types/model.py +++ b/src/twinkle_client/types/model.py @@ -1,344 +1,446 @@ # Copyright (c) ModelScope Contributors. All rights reserved. +"""Request / response models for the twinkle-native model endpoints. + +One declaration per endpoint, shared by Twinkle_Client and the server handler, so +there is a single answer to "what may this endpoint receive". Every field carries a +role (see :mod:`twinkle_client.types.base`): + +- plain fields are **control** fields: the handler consumes them or passes them as a + named argument, and they are never re-forwarded through ``**backend_kwargs``; +- :func:`backend_kwarg` / :func:`backend_only` fields are forwarded to the backend + when their value is not ``None``; +- :func:`passthrough` fields are declared dicts whose keys are dynamic (plugin + constructor / loss arguments) and are flattened into the backend kwargs. + +Requests are strict: an unknown top-level field is a typo and fails with 422 before +the task is enqueued. That is only safe because dynamic parameters have a declared +home -- the passthrough regions -- so strictness never blocks a legitimate +user-supplied argument. """ -Pydantic request/response models for twinkle model management endpoints. +from __future__ import annotations -These models are used by both the server-side handler and the twinkle client. -""" -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import Field, JsonValue, field_validator, model_validator from typing import Any, Dict, List, Optional, Union +from .base import ResponseModel, StrictRequest, backend_kwarg, backend_only, passthrough from .component import DataRef +from .data import WireInputBatch + + +class CreateRequest(StrictRequest): + """Body of ``POST /twinkle/create``: a session-establishing no-op.""" + +# --------------------------------------------------------------------------- # +# Control-plane requests +# --------------------------------------------------------------------------- # -class CreateRequest(BaseModel): - class Config: - extra = 'allow' +class AdapterRequest(StrictRequest): + """The shared shape of an adapter-scoped operation. + ``seq_id`` is an idempotency key, not a backend parameter: the submit shell + claims ``(session, adapter, seq_id)`` before enqueueing so a retried + gradient-mutating call is applied at most once. It must stay a declared field -- + under ``extra='forbid'`` an undeclared ``seq_id`` would be rejected outright, + which would silently disable that dedup. + """ -class ForwardRequest(BaseModel): - inputs: Any adapter_name: str - # Optional monotonic dedup key; only set by grad-mutating client calls (forward_backward). seq_id: Optional[int] = None + gradient_accumulation_steps: Optional[int] = backend_kwarg(default=None, ge=1) - class Config: - extra = 'allow' +class StepRequest(AdapterRequest): + """Body of ``POST /twinkle/step``.""" -class ForwardOnlyRequest(BaseModel): - inputs: Any - adapter_name: Optional[str] = None + optim_params: Optional[Dict[str, JsonValue]] = backend_kwarg(default=None) - class Config: - extra = 'allow' +class LrStepRequest(AdapterRequest): + """Body of ``POST /twinkle/lr_step``.""" -class DataPlaneForwardRequest(BaseModel): - input_refs: List[DataRef] = Field(min_length=1) - input_field: str | None = None - kwarg_fields: Dict[str, str] = Field(default_factory=dict) - adapter_name: str - # Optional monotonic dedup key; only set by grad-mutating data-plane calls. - seq_id: Optional[int] = None + # ``OptimizerParamScheduler.step(increment=...)``; the transformers scheduler has + # no equivalent knob. + increment: Optional[int] = backend_only('megatron', default=None, ge=0) - class Config: - extra = 'allow' +class ClipGradNormRequest(AdapterRequest): + """Body of ``POST /twinkle/clip_grad_norm``. -class DataPlaneForwardOnlyRequest(DataPlaneForwardRequest): - output_ref: DataRef | None = None - output_fields: Dict[str, str] = Field(default_factory=dict) + Bound to its own model rather than the bare :class:`AdapterRequest`: the endpoint + has always read these two values, and sharing a model with the parameterless ops + meant the schema could not say so. + """ - @model_validator(mode='after') - def validate_output(self) -> 'DataPlaneForwardOnlyRequest': - if (self.output_ref is None) != (len(self.output_fields) == 0): - raise ValueError('output_ref and output_fields must be configured together') - return self + max_grad_norm: float = Field(default=1.0, gt=0) + norm_type: int = Field(default=2, gt=0) -class AdapterRequest(BaseModel): - adapter_name: str - # Optional monotonic dedup key; only set by grad-mutating client calls (backward/step/lr_step). - seq_id: Optional[int] = None +class ClipGradAndStepRequest(ClipGradNormRequest): + """Body of ``POST /twinkle/clip_grad_and_step``.""" - class Config: - extra = 'allow' + optim_params: Optional[Dict[str, JsonValue]] = backend_kwarg(default=None) -class SetLossRequest(BaseModel): - loss_cls: str +class CalculateMetricRequest(StrictRequest): + """Body of ``POST /twinkle/calculate_metric``.""" + adapter_name: str + is_training: bool = True - class Config: - extra = 'allow' +# --------------------------------------------------------------------------- # +# Inline forward family +# +# Three endpoints, three models. They were one shared model, which meant the schema +# could not express that only the gradient-mutating variants take ``seq_id`` or that +# ``forward_only`` does not need an adapter -- the handler had to carry that +# knowledge instead, as a second source of truth. +# +# ``ForwardBackwardRequest`` is deliberately NOT the name of the fwd-bwd model: +# ``tinker.types.ForwardBackwardRequest`` already exists and two handlers import a +# module named ``types`` from each package, so a same-named model would be +# distinguishable only by import alias. +# --------------------------------------------------------------------------- # + + +class _InlineForwardBase(StrictRequest): + """Fields common to the three inline forward endpoints.""" + + inputs: WireInputBatch + task: Optional[str] = backend_kwarg(default=None) + temperature: Optional[float] = backend_kwarg(default=None, gt=0) + return_logits: Optional[bool] = backend_kwarg(default=None) + micro_batch_size: Optional[int] = backend_kwarg(default=None, ge=1) + gradient_accumulation_steps: Optional[int] = backend_kwarg(default=None, ge=1) + # Read only by the transformers backend. + sampling_masks: Optional[JsonValue] = backend_only('transformers', default=None) + router_replay_action: Optional[str] = backend_only('transformers', default=None) + # Loss inputs (``advantages`` / ``old_logps`` / ``ref_outputs`` / ...). Their key + # set is decided by the configured Loss, so they get a declared dict rather than + # top-level fields; the flattening in ``backend_kwargs`` keeps the backend call + # shape identical to before. + loss_kwargs: Dict[str, JsonValue] = passthrough() + + +class ForwardRequest(_InlineForwardBase): + """Body of ``POST /twinkle/forward``: keeps the graph, mutates no gradients.""" -class SetOptimizerRequest(BaseModel): - optimizer_cls: str adapter_name: str + disable_lora: Optional[bool] = backend_kwarg(default=None) - class Config: - extra = 'allow' +class ForwardOnlyRequest(_InlineForwardBase): + """Body of ``POST /twinkle/forward_only``: no graph, no gradients. -class SetLrSchedulerRequest(BaseModel): - scheduler_cls: str - adapter_name: str + ``adapter_name`` is optional here -- a reference forward may run against the base + weights -- and there is no ``seq_id`` because nothing is mutated to be idempotent + about. + """ + + adapter_name: Optional[str] = None + disable_lora: Optional[bool] = backend_kwarg(default=None) - class Config: - extra = 'allow' +class ForwardBackwardTaskRequest(_InlineForwardBase): + """Body of ``POST /twinkle/forward_backward``: accumulates gradients.""" -class SaveRequest(BaseModel): adapter_name: str - save_optimizer: bool = False - name: Optional[str] = None - is_sampler: bool = False # If True, delete existing sampler weights before saving + seq_id: Optional[int] = None + sync_gradients: Optional[bool] = backend_kwarg(default=None) + loss_scale: Optional[float] = backend_kwarg(default=None) - class Config: - extra = 'allow' +# --------------------------------------------------------------------------- # +# Data-plane forward family +# +# ``input_refs`` / ``input_field`` / ``kwarg_fields`` are control fields: the handler +# resolves them into rows and bound kwargs. No wire schema applies to a ``DataRef`` -- +# it is an opaque handle and the rows it points at never travel in this body. +# --------------------------------------------------------------------------- # -class UploadToHubRequest(BaseModel): - checkpoint_dir: Union[str, Dict] - hub_model_id: str - hub_token: Optional[str] = None - async_upload: bool = False - @field_validator('checkpoint_dir', mode='before') - @classmethod - def extract_checkpoint_dir(cls, v): - if isinstance(v, dict): - return v['twinkle_path'] - return v +class DataPlaneForwardRequest(StrictRequest): + """Body of the ``*_from_data_plane`` forward endpoints.""" - class Config: - extra = 'allow' + input_refs: List[DataRef] = Field(min_length=1) + input_field: Optional[str] = None + # Values are *field paths*, not parameter values, so this is not a passthrough + # region: nothing in it is forwarded verbatim. + kwarg_fields: Dict[str, str] = Field(default_factory=dict) + adapter_name: str + seq_id: Optional[int] = None + task: Optional[str] = backend_kwarg(default=None) + temperature: Optional[float] = backend_kwarg(default=None, gt=0) + return_logits: Optional[bool] = backend_kwarg(default=None) + disable_lora: Optional[bool] = backend_kwarg(default=None) + micro_batch_size: Optional[int] = backend_kwarg(default=None, ge=1) + gradient_accumulation_steps: Optional[int] = backend_kwarg(default=None, ge=1) + loss_kwargs: Dict[str, JsonValue] = passthrough() -class LoadRequest(BaseModel): - adapter_name: str - load_optimizer: bool = False - name: str +class DataPlaneForwardOnlyRequest(DataPlaneForwardRequest): + """Body of ``POST /twinkle/forward_only_from_data_plane``.""" - class Config: - extra = 'allow' + output_ref: Optional[DataRef] = None + output_fields: Dict[str, str] = Field(default_factory=dict) + @model_validator(mode='after') + def validate_output(self) -> 'DataPlaneForwardOnlyRequest': + if (self.output_ref is None) != (len(self.output_fields) == 0): + raise ValueError('output_ref and output_fields must be configured together') + return self -class ResumeFromCheckpointRequest(BaseModel): - """Request for /resume_from_checkpoint endpoint.""" - name: str - adapter_name: str = '' - resume_only_model: bool = False - class Config: - extra = 'allow' +# --------------------------------------------------------------------------- # +# Plugin setters +# +# Each takes the plugin identifier as a control field (the handler passes it +# positionally) plus one passthrough region for the plugin's constructor arguments. +# The passthrough keys are forwarded to the plugin as given -- there is no spelling +# check against a sibling ``target``: signature reflection cannot see a parameter a +# plugin reads straight out of ``**kwargs`` (``InputProcessor`` does this with +# ``padding_side``), so any such check rejects valid requests. A misspelt argument +# therefore surfaces from the plugin itself. +# --------------------------------------------------------------------------- # -class AddAdapterRequest(BaseModel): +class SetLossRequest(StrictRequest): + loss_cls: str adapter_name: str - # ``config`` is None for full-parameter training (no LoRA adapter) and a - # serialized LoraConfig string for LoRA training. - config: Optional[str] = None - save_dir: Optional[str] = None + init_kwargs: Dict[str, JsonValue] = passthrough() + - class Config: - extra = 'allow' +class SetOptimizerRequest(StrictRequest): + optimizer_cls: str + adapter_name: str + init_kwargs: Dict[str, JsonValue] = passthrough() -class SetTemplateRequest(BaseModel): - template_cls: str +class SetLrSchedulerRequest(StrictRequest): + scheduler_cls: str adapter_name: str + init_kwargs: Dict[str, JsonValue] = passthrough() - class Config: - extra = 'allow' +class SetTemplateRequest(StrictRequest): + """Body of ``POST /twinkle/set_template``. -class SetProcessorRequest(BaseModel): + No top-level ``model_id``: the backend always overrides it with its own + ``tokenizer_id``, so a declared field would advertise a parameter that has no + effect. Callers that pass ``model_id`` reach the template constructor through + ``init_kwargs`` like any other template argument. + """ + + template_cls: str + adapter_name: str + init_kwargs: Dict[str, JsonValue] = passthrough() + + +class SetProcessorRequest(StrictRequest): processor_cls: str adapter_name: str + init_kwargs: Dict[str, JsonValue] = passthrough() - class Config: - extra = 'allow' + +class AddMetricRequest(StrictRequest): + metric_cls: str + adapter_name: str + is_training: Optional[bool] = None + init_kwargs: Dict[str, JsonValue] = passthrough() -class CalculateMetricRequest(BaseModel): +class ApplyPatchRequest(StrictRequest): + patch_cls: str adapter_name: str - is_training: bool = True + init_kwargs: Dict[str, JsonValue] = passthrough() - class Config: - extra = 'allow' +# --------------------------------------------------------------------------- # +# Checkpoint I/O and adapter lifecycle +# --------------------------------------------------------------------------- # -class GetStateDictRequest(BaseModel): - adapter_name: str - class Config: - extra = 'allow' +class SaveRequest(StrictRequest): + adapter_name: str + name: Optional[str] = None + save_optimizer: bool = False + is_sampler: bool = False # If True, delete existing sampler weights before saving + consumed_train_samples: Optional[int] = backend_kwarg(default=None, ge=0) + merge_lora: Optional[bool] = backend_only('megatron', default=None) -class ClipGradAndStepRequest(BaseModel): +class LoadRequest(StrictRequest): adapter_name: str - max_grad_norm: float = 1.0 - norm_type: int = 2 - # Optional monotonic dedup key; set by the grad-mutating clip_grad_and_step call. - seq_id: Optional[int] = None + name: str + load_optimizer: bool = False + no_load_optim: Optional[bool] = backend_only('megatron', default=None) + no_load_rng: Optional[bool] = backend_only('megatron', default=None) + strict: Optional[bool] = backend_only('transformers', default=None) - class Config: - extra = 'allow' +class ResumeFromCheckpointRequest(StrictRequest): + """Body of ``POST /twinkle/resume_from_checkpoint``.""" -class ApplyPatchRequest(BaseModel): - patch_cls: str + name: str + adapter_name: str = '' + resume_only_model: bool = False + + +class AddAdapterRequest(StrictRequest): adapter_name: str + # ``config`` is None for full-parameter training (no LoRA adapter) and a + # serialized LoraConfig string for LoRA training. + config: Optional[str] = None + save_dir: Optional[str] = None + gradient_accumulation_steps: Optional[int] = backend_kwarg(default=None, ge=1) + init_kwargs: Dict[str, JsonValue] = passthrough() - class Config: - extra = 'allow' +class UploadToHubRequest(StrictRequest): + """Body of ``POST /twinkle/upload_to_hub``. -class AddMetricRequest(BaseModel): - metric_cls: str - adapter_name: str - is_training: Optional[bool] = None + No ``async_upload``: the server always runs the upload as a background task and + the client waits through the future layer, so the flag could only ever be ignored. + """ + + checkpoint_dir: Union[str, Dict[str, Any]] + hub_model_id: str + hub_token: Optional[str] = None - class Config: - extra = 'allow' + @field_validator('checkpoint_dir', mode='before') + @classmethod + def extract_checkpoint_dir(cls, v): + """Accept a ``save`` response dict and take its twinkle path. + + Raises a validation error -- not ``KeyError`` -- when the key is absent, so a + wrong-shaped dict is a 422 naming the missing key instead of a 500. + """ + if isinstance(v, dict): + if 'twinkle_path' not in v: + raise ValueError("checkpoint_dir dict must contain 'twinkle_path'") + return v['twinkle_path'] + return v -# --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- # # Response models -# --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- # -class OkResponse(BaseModel): +class OkResponse(ResponseModel): """Response for endpoints whose underlying method returns None.""" status: str = 'ok' -class ModelResult(BaseModel): +class ModelResult(ResponseModel): """Generic single-value result wrapper returned by result-bearing endpoints.""" result: Any # --- Result-bearing responses --- -class ForwardResponse(BaseModel): + +class ForwardResponse(ResponseModel): """Response for /forward and /forward_only endpoints (returns ModelOutput).""" result: Any -class ForwardBackwardResponse(BaseModel): +class ForwardBackwardResponse(ResponseModel): """Response for /forward_backward endpoint (returns ModelOutput).""" result: Any -class CalculateLossResponse(BaseModel): +class CalculateLossResponse(ResponseModel): """Response for /calculate_loss endpoint (returns float).""" result: float -class ClipGradNormResponse(BaseModel): +class ClipGradNormResponse(ResponseModel): """Response for /clip_grad_norm endpoint (returns float as str).""" result: str -class GetTrainConfigsResponse(BaseModel): +class GetTrainConfigsResponse(ResponseModel): """Response for /get_train_configs endpoint (returns str).""" result: str -class CalculateMetricResponse(BaseModel): +class CalculateMetricResponse(ResponseModel): """Response for /calculate_metric endpoint (returns Dict).""" result: Dict[str, Any] -class SaveResponse(BaseModel): +class SaveResponse(ResponseModel): """Response for /save endpoint (returns twinkle path + checkpoint dir).""" twinkle_path: str checkpoint_dir: Optional[str] = None -class TrainingProgressResponse(BaseModel): +class TrainingProgressResponse(ResponseModel): """Response for /resume_from_checkpoint endpoint.""" result: Dict[str, Any] # --- Void responses (return None → OkResponse) --- + class BackwardResponse(OkResponse): """Response for /backward endpoint.""" - pass class StepResponse(OkResponse): """Response for /step (optimizer step) endpoint.""" - pass class ZeroGradResponse(OkResponse): """Response for /zero_grad endpoint.""" - pass class LrStepResponse(OkResponse): """Response for /lr_step endpoint.""" - pass class SetLossResponse(OkResponse): """Response for /set_loss endpoint.""" - pass class SetOptimizerResponse(OkResponse): """Response for /set_optimizer endpoint.""" - pass class SetLrSchedulerResponse(OkResponse): """Response for /set_lr_scheduler endpoint.""" - pass class LoadResponse(OkResponse): """Response for /load endpoint.""" - pass class SetTemplateResponse(OkResponse): """Response for /set_template endpoint.""" - pass class SetProcessorResponse(OkResponse): """Response for /set_processor endpoint.""" - pass class ClipGradAndStepResponse(OkResponse): """Response for /clip_grad_and_step endpoint.""" - pass class ApplyPatchResponse(OkResponse): """Response for /apply_patch endpoint.""" - pass class AddMetricResponse(OkResponse): """Response for /add_metric endpoint.""" - pass # --- Other responses --- -class CreateResponse(BaseModel): - """Response for /create endpoint.""" - status: str = 'ok' - -class AddAdapterResponse(BaseModel): - """Response for /add_adapter_to_model endpoint.""" +class CreateResponse(ResponseModel): + """Response for /create endpoint.""" status: str = 'ok' - adapter_name: str diff --git a/src/twinkle_client/types/processor.py b/src/twinkle_client/types/processor.py index fe8674ce..7fdb1a64 100644 --- a/src/twinkle_client/types/processor.py +++ b/src/twinkle_client/types/processor.py @@ -1,46 +1,55 @@ # Copyright (c) ModelScope Contributors. All rights reserved. +"""Request / response models for the twinkle processor endpoints. + +The processor surface is a generic RPC bridge: ``create`` names a class to build and +``call`` names a method to invoke, both with caller-supplied arguments whose names are +only known to the target. Those arguments therefore live in declared passthrough +dicts (``init_kwargs`` / ``call_kwargs``) rather than being spread over the top level, +which is what lets the envelope itself be strict -- a misspelt ``processor_id`` or +``function`` now fails instead of being silently treated as an argument. + +No ``target``: the callable is resolved from ``processor_type`` + ``class_type`` (or a +live instance plus ``function``), not from a single sibling field, so the passthrough +keys are forwarded unchecked. Claiming otherwise would need a second resolution path +that guesses. + +Class names are prefixed with ``Processor`` to avoid collisions when importing from +``twinkle_client.types`` alongside ``model.py``. """ -Pydantic request/response models for twinkle processor endpoints. +from __future__ import annotations -These models are used by both the server-side handler and the twinkle client. +from pydantic import JsonValue +from typing import Any, Dict -Note: Class names are prefixed with 'Processor' to avoid name collisions when -importing from twinkle_client.types alongside model.py classes. -""" -from pydantic import BaseModel -from typing import Any +from .base import ResponseModel, StrictRequest, passthrough -class ProcessorCreateRequest(BaseModel): +class ProcessorCreateRequest(StrictRequest): processor_type: str class_type: str - - class Config: - extra = 'allow' + init_kwargs: Dict[str, JsonValue] = passthrough() -class ProcessorHeartbeatRequest(BaseModel): +class ProcessorHeartbeatRequest(StrictRequest): processor_id: str -class ProcessorCallRequest(BaseModel): +class ProcessorCallRequest(StrictRequest): processor_id: str function: str - - class Config: - extra = 'allow' + call_kwargs: Dict[str, JsonValue] = passthrough() -class ProcessorCreateResponse(BaseModel): +class ProcessorCreateResponse(ResponseModel): """Response body for the /create endpoint.""" processor_id: str -class ProcessorHeartbeatResponse(BaseModel): +class ProcessorHeartbeatResponse(ResponseModel): """Response body for the /heartbeat endpoint.""" status: str = 'ok' -class ProcessorCallResponse(BaseModel): +class ProcessorCallResponse(ResponseModel): """Response body for the /call endpoint.""" result: Any diff --git a/src/twinkle_client/types/sampler.py b/src/twinkle_client/types/sampler.py index 47ce1bdd..5e5207c1 100644 --- a/src/twinkle_client/types/sampler.py +++ b/src/twinkle_client/types/sampler.py @@ -1,26 +1,49 @@ # Copyright (c) ModelScope Contributors. All rights reserved. +"""Request / response models for the twinkle-native sampler endpoints. + +Shared by the server handler and the twinkle client. Field roles follow +:mod:`twinkle_client.types.base`; the sampler handlers pass everything they need +explicitly, so these requests carry control fields and -- for the template setter -- +one passthrough region, and no free-floating backend kwargs. + +Class names carry a ``Sampler`` prefix wherever ``model.py`` already owns the bare +name (``AddAdapterRequest``, ``SetTemplateRequest``, ``CreateResponse`` and their +responses). The two modules describe *different* endpoints with different field sets; +a shared bare name is distinguished only by an import alias and, when a handler does +``import twinkle_client.types as types``, silently resolves to whichever module the +package ``__init__`` re-exported first -- which is how the sampler endpoints once +bound ``model.py``'s schema. Prefixing at the definition site removes the ambiguity, +matching :mod:`twinkle_client.types.processor`. """ -Pydantic request/response models for twinkle sampler endpoints. +from __future__ import annotations -These models are used by both the server-side handler and the twinkle client. -""" -from pydantic import BaseModel, Field +from pydantic import Field, JsonValue from typing import Any, Dict, List, Literal, Optional, Tuple +from .base import ResponseModel, StrictRequest, passthrough +from .data import WireInputBatch + StopReason = Literal['length', 'stop', 'abort', 'error'] -class SampleRequest(BaseModel): - """Request body for the /sample endpoint.""" - inputs: Any = Field(..., description='List of Trajectory or InputFeature dicts') - sampling_params: Optional[Dict[str, Any]] = Field( +class SampleRequest(StrictRequest): + """Request body for the ``/sample`` and ``/sample_stream`` endpoints. + + ``num_samples`` is not a top-level field: it is a sampling parameter and + ``SamplingParams.from_dict(sampling_params)`` is the one place sampling + parameters are built. A second, top-level spelling would be a second source of + truth for the same value. + """ + + inputs: WireInputBatch = Field(..., description='Trajectory or InputFeature entries to sample from') + sampling_params: Optional[Dict[str, JsonValue]] = Field( None, description='Sampling parameters (max_tokens, temperature, num_samples, etc.)') adapter_name: str = Field('', description='Adapter name for LoRA inference') adapter_uri: Optional[str] = Field( None, description='Adapter URI (twinkle:// path or local path) for LoRA inference') -class SampledSequenceModel(BaseModel): +class SampledSequenceModel(ResponseModel): """A single sampled sequence, mirroring twinkle.data_format.SampledSequence.""" stop_reason: StopReason = Field(..., description="Stop reason: 'length' or 'stop'") tokens: List[int] = Field(..., description='Token IDs of the sampled sequence') @@ -30,7 +53,7 @@ class SampledSequenceModel(BaseModel): None, description='Updated InputFeature after sampling (input_ids, labels, etc.)') -class SampleResponseModel(BaseModel): +class SampleResponseModel(ResponseModel): """Mirroring twinkle.data_format.SampleResponse.""" sequences: List[SampledSequenceModel] = Field( ..., description='List of sampled sequences') @@ -40,37 +63,35 @@ class SampleResponseModel(BaseModel): topk_prompt_logprobs: Optional[List[Optional[List[Tuple[int, float]]]]] = None -class SampleResponseModelList(BaseModel): +class SampleResponseModelList(ResponseModel): """Response body for the /sample endpoint""" samples: List[SampleResponseModel] = Field(..., description='List of sample responses') -class SetTemplateRequest(BaseModel): - """Request body for the /set_template endpoint.""" +class SamplerSetTemplateRequest(StrictRequest): + """Request body for the sampler ``/set_template`` endpoint.""" template_cls: str = Field(..., description="Template class name (e.g. 'Template')") adapter_name: str = Field('', description='Adapter name to associate the template with') - - class Config: - extra = 'allow' + init_kwargs: Dict[str, JsonValue] = passthrough() -class SetTemplateResponse(BaseModel): - """Response body for the /set_template endpoint.""" +class SamplerSetTemplateResponse(ResponseModel): + """Response body for the sampler /set_template endpoint.""" status: str = 'ok' -class AddAdapterRequest(BaseModel): - """Request body for the /add_adapter_to_sampler endpoint.""" +class SamplerAddAdapterRequest(StrictRequest): + """Request body for the ``/add_adapter_to_sampler`` endpoint.""" adapter_name: str = Field(..., description='Name of the adapter to add') config: Any = Field(..., description='LoRA configuration dict') -class AddAdapterResponse(BaseModel): +class SamplerAddAdapterResponse(ResponseModel): """Response body for the /add_adapter_to_sampler endpoint.""" status: str = 'ok' adapter_name: str -class CreateResponse(BaseModel): - """Response body for the /create endpoint.""" +class SamplerCreateResponse(ResponseModel): + """Response body for the sampler /create endpoint.""" status: str = 'ok' diff --git a/src/twinkle_client/types/server.py b/src/twinkle_client/types/server.py index 1c7c992d..235032a2 100644 --- a/src/twinkle_client/types/server.py +++ b/src/twinkle_client/types/server.py @@ -1,48 +1,49 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Shared Pydantic response models for the twinkle server health/error endpoints.""" from pydantic import BaseModel -from typing import Any, List, Optional +from typing import List + +from .base import ResponseModel, StrictRequest class SupportedModel(BaseModel): - """Information about a supported model.""" + """Information about a supported model. + + A nested value inside a response, not a response body of its own, so it keeps the + plain base -- the strict/ignore split applies to what crosses the wire as a whole. + """ model_name: str -class GetServerCapabilitiesResponse(BaseModel): +class GetServerCapabilitiesResponse(ResponseModel): """Response body for the /get_server_capabilities endpoint.""" supported_models: List[SupportedModel] -class HealthResponse(BaseModel): +class HealthResponse(ResponseModel): status: str -class DeleteCheckpointResponse(BaseModel): +class DeleteCheckpointResponse(ResponseModel): success: bool message: str -class ErrorResponse(BaseModel): +class ErrorResponse(ResponseModel): detail: str -class WeightsInfoRequest(BaseModel): +class WeightsInfoRequest(StrictRequest): twinkle_path: str -class WeightsInfoResponse(BaseModel): - """Response body for the /weights_info endpoint.""" - weights_info: Any - - -class CheckpointPathResponse(BaseModel): +class CheckpointPathResponse(ResponseModel): """Response body for the /checkpoint_path endpoint.""" path: str twinkle_path: str -class CapacityInfoResponse(BaseModel): +class CapacityInfoResponse(ResponseModel): """Response body for the /capacity_info endpoint.""" max_loras: int used_loras: int diff --git a/src/twinkle_client/types/session.py b/src/twinkle_client/types/session.py index f6b1adb7..da8795d5 100644 --- a/src/twinkle_client/types/session.py +++ b/src/twinkle_client/types/session.py @@ -1,24 +1,24 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Pydantic models for twinkle session management endpoints.""" -from pydantic import BaseModel from typing import Any, Dict, Optional +from .base import ResponseModel, StrictRequest -class CreateSessionRequest(BaseModel): + +class CreateSessionRequest(StrictRequest): """Request body for POST /twinkle/create_session.""" metadata: Optional[Dict[str, Any]] = None -class CreateSessionResponse(BaseModel): +class CreateSessionResponse(ResponseModel): """Response body for POST /twinkle/create_session.""" session_id: str -class SessionHeartbeatRequest(BaseModel): +class SessionHeartbeatRequest(StrictRequest): """Request body for POST /twinkle/session_heartbeat.""" session_id: str -class SessionHeartbeatResponse(BaseModel): +class SessionHeartbeatResponse(ResponseModel): """Response body for POST /twinkle/session_heartbeat.""" - pass diff --git a/src/twinkle_client/types/training.py b/src/twinkle_client/types/training.py index bab59ac6..da2c7d67 100644 --- a/src/twinkle_client/types/training.py +++ b/src/twinkle_client/types/training.py @@ -9,6 +9,8 @@ from pydantic import BaseModel from typing import Any, Dict, List, Optional +from .base import ResponseModel + class Cursor(BaseModel): limit: int @@ -49,12 +51,12 @@ class TrainingRun(BaseModel): user_metadata: Optional[Dict[str, Any]] = None -class TrainingRunsResponse(BaseModel): +class TrainingRunsResponse(ResponseModel): training_runs: List[TrainingRun] cursor: Cursor -class CheckpointsListResponse(BaseModel): +class CheckpointsListResponse(ResponseModel): checkpoints: List[Checkpoint] cursor: Optional[Cursor] = None @@ -68,7 +70,7 @@ class ParsedCheckpointTwinklePath(BaseModel): checkpoint_id: str -class WeightsInfoResponse(BaseModel): +class WeightsInfoResponse(ResponseModel): """Twinkle weights info response.""" training_run_id: str base_model: str diff --git a/src/twinkle_client/utils/__init__.py b/src/twinkle_client/utils/__init__.py new file mode 100644 index 00000000..bcd3b883 --- /dev/null +++ b/src/twinkle_client/utils/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Client-internal utilities. + +Regular package (carries this ``__init__``) so ``patch_tinker`` is included by +``setuptools.packages.find`` in a built wheel; a namespace-only directory would be +dropped from the distribution. +""" diff --git a/tests/server/contract/client_api_routes.json b/tests/server/contract/client_api_routes.json index 8af1e5c7..ec8f176e 100644 --- a/tests/server/contract/client_api_routes.json +++ b/tests/server/contract/client_api_routes.json @@ -393,7 +393,7 @@ }, "POST /twinkle/clip_grad_norm": { "body": [ - "AdapterRequest" + "ClipGradNormRequest" ], "response": "TaskEnvelope", "statusCode": 200 @@ -414,7 +414,7 @@ }, "POST /twinkle/forward_backward": { "body": [ - "ForwardRequest" + "ForwardBackwardTaskRequest" ], "response": "TaskEnvelope", "statusCode": 200 @@ -463,7 +463,7 @@ }, "POST /twinkle/lr_step": { "body": [ - "AdapterRequest" + "LrStepRequest" ], "response": "TaskEnvelope", "statusCode": 200 @@ -526,7 +526,7 @@ }, "POST /twinkle/step": { "body": [ - "AdapterRequest" + "StepRequest" ], "response": "TaskEnvelope", "statusCode": 200 @@ -572,9 +572,9 @@ }, "POST /twinkle/add_adapter_to_sampler": { "body": [ - "AddAdapterRequest" + "SamplerAddAdapterRequest" ], - "response": "AddAdapterResponse", + "response": "SamplerAddAdapterResponse", "statusCode": 200 }, "POST /twinkle/apply_patch": { @@ -586,7 +586,7 @@ }, "POST /twinkle/create": { "body": [], - "response": "CreateResponse", + "response": "SamplerCreateResponse", "statusCode": 200 }, "POST /twinkle/sample": { @@ -612,9 +612,9 @@ }, "POST /twinkle/set_template": { "body": [ - "SetTemplateRequest" + "SamplerSetTemplateRequest" ], - "response": "SetTemplateResponse", + "response": "SamplerSetTemplateResponse", "statusCode": 200 }, "POST /twinkle/unload_adapter_paths": { diff --git a/tests/server/lifecycle/test_run_submit_dedup.py b/tests/server/lifecycle/test_run_submit_dedup.py index a8861484..f9038d55 100644 --- a/tests/server/lifecycle/test_run_submit_dedup.py +++ b/tests/server/lifecycle/test_run_submit_dedup.py @@ -13,6 +13,7 @@ import pytest from twinkle.server.lifecycle.submit import run_submit +from twinkle_client.types.model import ForwardBackwardTaskRequest class _FakeState: @@ -38,6 +39,9 @@ class _FakeManagement: def __init__(self, record_after_claim): self.state = _FakeState(record_after_claim) self._task_queue_config = SimpleNamespace(effective_execution_timeout=60.0) + # A real deployment declares its backend; preflight reads it from here. + self.backend = 'transformers' + self.data_world_size = 1 async def _on_request_start(self, request): return 'token' @@ -51,8 +55,13 @@ def _request(): return SimpleNamespace(state=SimpleNamespace(session_id='sess-1', request_id='rq-1')) -def _body(): - return SimpleNamespace(adapter_name='ad', seq_id=7) +def _body(adapter_name: str = 'ad', seq_id: int = 7) -> ForwardBackwardTaskRequest: + """A real request model, not a stand-in. + + ``run_submit`` now reads field roles off the body to build the backend kwargs and to + run preflight, so a ``SimpleNamespace`` would exercise a shape production never sees. + """ + return ForwardBackwardTaskRequest(inputs=[{'input_ids': [1, 2]}], adapter_name=adapter_name, seq_id=seq_id) async def _call(self, body, adapter_name, token): # pragma: no cover - never invoked @@ -93,7 +102,7 @@ async def test_dedup_key_is_scoped_per_adapter(): await run_submit( mgmt, _request(), - SimpleNamespace(adapter_name=adapter, seq_id=1), + _body(adapter_name=adapter, seq_id=1), task_type='forward_backward', backend_call=_call) diff --git a/tests/server/validation/__init__.py b/tests/server/validation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/server/validation/test_preflight.py b/tests/server/validation/test_preflight.py new file mode 100644 index 00000000..81e5832b --- /dev/null +++ b/tests/server/validation/test_preflight.py @@ -0,0 +1,243 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Preflight: everything decidable about a request before it is enqueued. + +The property under test is not "an error is returned" but *where* it is returned. A +rejection that happens inside the queued task has already written a future record and +fanned the call out to every data-parallel rank; a rejection in preflight has done +neither. Each test therefore asserts on the side effects (future records, backend calls) +as well as the status code. +""" +from __future__ import annotations + +import pytest +from types import SimpleNamespace + +from twinkle.server.exceptions import RequestRejectedError +from twinkle.server.lifecycle.submit import backend_kwargs, run_submit +from twinkle.server.validation import BackendCapability, EndpointUnavailableError, assert_request_supported +from twinkle.server.validation.backend_compat import resolve_backend +from twinkle_client.types import model as model_types + + +class _Deployment: + """A deployment stub that records what the request managed to reach.""" + + def __init__(self, backend: str = 'transformers'): + self.backend = backend + self.data_world_size = 1 + self.futures: dict[str, dict] = {} + self.backend_calls: list = [] + self.claimed: list = [] + self._task_queue_config = SimpleNamespace(effective_execution_timeout=60.0) + self.state = SimpleNamespace( + claim_seq=self._claim_seq, + get_future=self._get_future, + release_seq=self._release_seq, + ) + + async def _claim_seq(self, dedup_key, request_id, ttl): + self.claimed.append(dedup_key) + return None + + async def _get_future(self, request_id): + return self.futures.get(request_id) + + async def _release_seq(self, dedup_key): + self.claimed.remove(dedup_key) + + async def _on_request_start(self, request): + return 'token' + + async def submit_and_peek(self, task, *, request_id, **kwargs): + self.futures[request_id] = {'status': 'queued'} + return await task() + + +def _request(): + return SimpleNamespace(state=SimpleNamespace(session_id='sess', request_id='rq')) + + +async def _call(self, body, adapter_name, token): + self.backend_calls.append(type(body).__name__) + return {'ok': True} + + +# --------------------------------------------------------------------------- # +# Backend resolution +# --------------------------------------------------------------------------- # + + +def test_backend_is_read_from_the_deployment_not_guessed(): + assert resolve_backend(_Deployment('megatron')) == 'megatron' + # A sampler deployment has no backend concept at all, and must not be invented. + assert resolve_backend(SimpleNamespace()) is None + + +# --------------------------------------------------------------------------- # +# Endpoint capability +# --------------------------------------------------------------------------- # + + +def test_megatron_rejects_the_split_gradient_endpoints_with_501(): + for capability in (BackendCapability.Forward, BackendCapability.Backward, BackendCapability.CalculateLoss): + with pytest.raises(EndpointUnavailableError) as raised: + assert_request_supported( + _Deployment('megatron'), model_types.AdapterRequest(adapter_name='a'), capability=capability) + assert raised.value.error_code == 501 + assert 'forward_backward' in str(raised.value), 'the alternative endpoints must be named' + + +def test_transformers_serves_the_split_gradient_endpoints(): + assert_request_supported( + _Deployment('transformers'), + model_types.AdapterRequest(adapter_name='a'), + capability=BackendCapability.Forward) + + +def test_mock_backend_serves_every_endpoint(): + """The mock is a test double; restricting it would only break tests.""" + assert_request_supported( + _Deployment('mock'), model_types.AdapterRequest(adapter_name='a'), capability=BackendCapability.Forward) + + +@pytest.mark.asyncio +async def test_capability_rejection_writes_no_future_and_calls_no_backend(): + deployment = _Deployment('megatron') + with pytest.raises(EndpointUnavailableError): + await run_submit( + deployment, + _request(), + model_types.AdapterRequest(adapter_name='a'), + task_type='backward', + backend_call=_call, + capability=BackendCapability.Backward) + assert deployment.futures == {}, 'a rejected request must leave no future record' + assert deployment.backend_calls == [], 'the backend must not run for a rejected request' + assert deployment.claimed == [], 'no seq claim may be taken before preflight passes' + + +# --------------------------------------------------------------------------- # +# Backend-only parameters +# --------------------------------------------------------------------------- # + + +def test_a_megatron_only_parameter_is_rejected_on_transformers(): + with pytest.raises(RequestRejectedError) as raised: + assert_request_supported(_Deployment('transformers'), model_types.SaveRequest(adapter_name='a', merge_lora=True)) + assert raised.value.error_code == 422 + assert 'merge_lora' in str(raised.value) + assert 'megatron' in str(raised.value) + + +def test_a_transformers_only_parameter_is_rejected_on_megatron(): + with pytest.raises(RequestRejectedError): + assert_request_supported( + _Deployment('megatron'), model_types.LoadRequest(adapter_name='a', name='ckpt', strict=True)) + + +def test_an_unset_backend_only_parameter_is_not_rejected(): + """Only a *sent* value is checked. + + This is why every restricted field is ``Optional[...] = None``: had one carried its + backend's own default, it would look sent on every request and the other half of the + fleet would reject everything. + """ + assert_request_supported(_Deployment('transformers'), model_types.SaveRequest(adapter_name='a')) + assert_request_supported(_Deployment('megatron'), model_types.LoadRequest(adapter_name='a', name='ckpt')) + + +def test_every_restricted_field_is_optional_with_a_none_default(): + from twinkle_client.types.base import FieldRole, fields_with_role, read_backend_only + offenders = [] + for model_cls in vars(model_types).values(): + if not isinstance(model_cls, type) or not hasattr(model_cls, 'model_fields'): + continue + for name, info in fields_with_role(model_cls, FieldRole.BackendKwarg).items(): + if read_backend_only(info) and info.get_default() is not None: + offenders.append(f'{model_cls.__name__}.{name}') + assert not offenders, ('these backend-restricted fields carry a non-None default, so "non-None means wrongly ' + f'targeted" would reject every request on the other backend: {offenders}') + + +# --------------------------------------------------------------------------- # +# Passthrough keys are forwarded, not judged +# --------------------------------------------------------------------------- # + + +def test_a_real_parameter_read_from_kwargs_is_not_rejected(): + """The case that removed the passthrough spelling check. + + ``InputProcessor`` declares ``padding_free`` and reads ``padding_side`` via + ``kwargs.get`` -- both are real parameters, and ``inspect.signature`` only sees the + first. A similarity check scored them 0.75 and rejected + ``set_processor('InputProcessor', padding_side='right')``, a call used throughout the + cookbook and the E2E suite. No threshold fixes that: it also has to catch + ``bate`` -> ``beta`` at 0.5. Rejecting valid requests is worse than missing a typo, so + passthrough contents are forwarded unjudged. + """ + assert_request_supported( + _Deployment(), + model_types.SetProcessorRequest( + processor_cls='InputProcessor', adapter_name='a', init_kwargs={'padding_side': 'right'})) + + +def test_an_unrecognised_plugin_argument_is_forwarded(): + """Plugins accept ``**kwargs``, so "unknown" cannot mean "wrong".""" + assert_request_supported( + _Deployment(), + model_types.SetLossRequest(loss_cls='DPOLoss', adapter_name='a', init_kwargs={'my_custom_knob': 1})) + + +def test_no_plugin_download_happens_during_validation(monkeypatch): + """A validation path must have no side effects, and resolving a remote id downloads.""" + import twinkle.utils.loader as loader + monkeypatch.setattr(loader.Plugin, 'load_plugin', + lambda *a, **k: pytest.fail('validation must not download a plugin')) + assert_request_supported( + _Deployment(), + model_types.SetLossRequest(loss_cls='ms://someone/MyLoss', adapter_name='a', init_kwargs={'beta': 0.1})) + + +# --------------------------------------------------------------------------- # +# Forwarding +# --------------------------------------------------------------------------- # + + +def test_control_fields_are_never_forwarded_to_the_backend(): + """``inputs`` / ``adapter_name`` / ``seq_id`` are already passed explicitly. + + Forwarding them again would duplicate a keyword argument, or leak a protocol field + into a backend signature. + """ + body = model_types.ForwardBackwardTaskRequest( + inputs=[{'input_ids': [1, 2]}], adapter_name='a', seq_id=3, task='embedding') + assert backend_kwargs(body) == {'task': 'embedding'} + + +def test_unset_backend_parameters_are_not_forwarded(): + body = model_types.ForwardRequest(inputs=[{'input_ids': [1]}], adapter_name='a') + assert backend_kwargs(body) == {} + + +def test_passthrough_contents_are_flattened(): + body = model_types.ForwardBackwardTaskRequest( + inputs=[{'input_ids': [1]}], adapter_name='a', loss_kwargs={'advantages': [0.5]}) + assert backend_kwargs(body) == {'advantages': [0.5]} + + +def test_a_passthrough_key_shadowing_a_declared_parameter_is_an_error(): + """Silently letting one win would make the effective value depend on merge order.""" + body = model_types.ForwardRequest(inputs=[{'input_ids': [1]}], adapter_name='a', task='causal_lm', + loss_kwargs={'task': 'embedding'}) + with pytest.raises(ValueError, match='collides'): + backend_kwargs(body) + + +def test_plugin_identifiers_are_not_forwarded_as_kwargs(): + """The handler passes ``loss_cls`` positionally; forwarding it too would duplicate it.""" + body = model_types.SetLossRequest(loss_cls='DPOLoss', adapter_name='a', init_kwargs={'beta': 0.1}) + assert backend_kwargs(body) == {'beta': 0.1} + + +if __name__ == '__main__': + raise SystemExit(pytest.main([__file__, '-v'])) diff --git a/tests/server/validation/test_request_wire.py b/tests/server/validation/test_request_wire.py new file mode 100644 index 00000000..0c346e32 --- /dev/null +++ b/tests/server/validation/test_request_wire.py @@ -0,0 +1,234 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Level 1: what a malformed request body looks like on the wire. + +Driven through FastAPI's ``TestClient`` against the real route table, so what is asserted +is the response a client actually receives -- not a schema call in isolation. A rejection +here happens during body parsing, before any handler runs, which is what makes it free of +queue and backend side effects. +""" +from __future__ import annotations + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pydantic import ValidationError + +from twinkle.server.validation.errors import register_validation_error_handler +from twinkle_client.types import model as model_types +from twinkle_client.types.base import StrictRequest + + +@pytest.fixture(scope='module') +def client() -> TestClient: + """An app carrying the real request models and the shared error handler. + + Handlers are stubs on purpose: the point is that a bad body never reaches one, so a + stub that records nothing is the strongest possible witness -- if it is invoked, the + check did not happen. + """ + app = FastAPI() + register_validation_error_handler(app) + + @app.post('/forward') + async def forward(body: model_types.ForwardRequest): + return {'reached_handler': True} + + @app.post('/set_loss') + async def set_loss(body: model_types.SetLossRequest): + return {'reached_handler': True} + + @app.post('/save') + async def save(body: model_types.SaveRequest): + return {'reached_handler': True} + + @app.post('/forward_backward') + async def forward_backward(body: model_types.ForwardBackwardTaskRequest): + return {'reached_handler': True} + + return TestClient(app) + + +def _post(client: TestClient, path: str, body: dict): + return client.post(path, json=body) + + +def _valid_forward(**overrides) -> dict: + return {'inputs': [{'input_ids': [1, 2, 3]}], 'adapter_name': 'a', **overrides} + + +def test_a_valid_body_reaches_the_handler(client): + response = _post(client, '/forward', _valid_forward()) + assert response.status_code == 200 + assert response.json() == {'reached_handler': True} + + +def test_an_unknown_top_level_field_is_rejected(client): + response = _post(client, '/forward', _valid_forward(adapter_nmae='typo')) + assert response.status_code == 422 + body = response.json() + assert body['category'] == 'user' + assert body['error_code'] == 422 + assert 'adapter_nmae' in body['error'] + assert any(detail['field'] == 'adapter_nmae' for detail in body['details']) + assert 'reached_handler' not in body + + +def test_the_error_names_the_client_version_mismatch(client): + """An unknown top-level field is exactly what an outdated client looks like.""" + response = _post(client, '/forward', _valid_forward(advantages=[0.1])) + assert response.status_code == 422 + assert 'upgrade' in response.json()['error'].lower() + + +def test_the_error_body_is_an_error_payload_not_fastapi_detail(client): + """One error shape on the wire, or a client has to learn two.""" + body = _post(client, '/forward', _valid_forward(unknown=1)).json() + assert set(body) >= {'error', 'category', 'error_code', 'request_id'} + assert 'detail' not in body + + +def test_no_traceback_is_returned(client): + """A rejected body is the caller's problem, not a crash to be dumped at them.""" + assert 'traceback' not in _post(client, '/forward', _valid_forward(unknown=1)).json() + + +def test_details_locate_the_field_inside_the_body(client): + body = _post(client, '/forward', {'inputs': [{'input_ids': [1.5]}], 'adapter_name': 'a'}).json() + assert body['error_code'] == 422 + assert any('inputs' in detail['path'] for detail in body['details']) + + +def test_a_token_in_the_body_is_rejected(client): + """``token`` comes from the Authorization header only. + + Declaring it as a field would make a body-supplied token *legal* under + ``extra='forbid'``, which is a credential-forgery path, not a convenience. + """ + assert _post(client, '/forward', _valid_forward(token='stolen')).status_code == 422 + assert 'token' not in model_types.ForwardRequest.model_fields + + +def test_seq_id_is_still_accepted(client): + """Strictness must not break the retry idempotency key. + + Were ``seq_id`` undeclared, ``extra='forbid'`` would reject every retried + gradient-mutating call -- disabling the dedup that prevents a double-apply. It is + declared on the gradient-mutating models only, which is where the client sends it. + """ + assert _post(client, '/forward_backward', _valid_forward(seq_id=4)).status_code == 200 + assert 'seq_id' in model_types.ForwardBackwardTaskRequest.model_fields + assert 'seq_id' in model_types.AdapterRequest.model_fields + assert 'seq_id' in model_types.DataPlaneForwardRequest.model_fields + + +def test_a_dynamic_plugin_argument_is_accepted_inside_its_region(client): + """Strictness at the top level, freedom inside the declared dict.""" + response = _post(client, '/set_loss', { + 'loss_cls': 'DPOLoss', + 'adapter_name': 'a', + 'init_kwargs': {'beta': 0.1, 'anything_at_all': [1, 2]}, + }) + assert response.status_code == 200 + + +def test_a_non_json_value_fails_in_the_client_before_any_request(): + """``JsonValue`` earns its keep at Level 0, not Level 1. + + Anything that arrived as JSON is by definition a JSON value, so this annotation can + only ever reject something in the caller's own process -- which is the useful place, + because the caller still has the offending object and a stack trace pointing at it. + """ + from pydantic import ValidationError + with pytest.raises(ValidationError) as raised: + model_types.SetLossRequest(loss_cls='DPOLoss', adapter_name='a', init_kwargs={'beta': object()}) + assert 'init_kwargs' in str(raised.value), 'the error must name the field that holds the bad value' + + +def test_a_checkpoint_dict_without_its_key_is_a_422_not_a_500(client): + """It used to surface as ``KeyError`` -> 500, blaming the server for a bad body.""" + response = _post(client, '/save', {'adapter_name': 'a', 'checkpoint_dir': {'wrong': 'shape'}}) + assert response.status_code == 422 + + +# --------------------------------------------------------------------------- # +# Coverage: no twinkle-native route may keep a lax body +# --------------------------------------------------------------------------- # + +# The one twinkle-native route with no request body at all. +_BODYLESS = {('model', 'GET /healthz')} + + +def test_every_twinkle_route_body_is_strict(): + """Enumerated from the live route table, not from a hardcoded count. + + A count would have to be updated by whoever adds a route -- exactly the person who + would also forget the base class. + """ + from fastapi.routing import APIRoute + from tests.server.contract.client_api_harness import build_model_app, build_processor_app, build_sampler_app + + offenders = [] + for app_name, builder in (('model', build_model_app), ('sampler', build_sampler_app), ('processor', + build_processor_app)): + for route in builder().routes: + if not isinstance(route, APIRoute): + continue + for method in sorted(route.methods & {'GET', 'POST', 'PUT', 'PATCH', 'DELETE'}): + key = f'{method} {route.path}' + if not route.path.startswith(('/twinkle', '/healthz')) or (app_name, key) in _BODYLESS: + continue + for field in route.dependant.body_params: + annotation = field.field_info.annotation + if not (isinstance(annotation, type) and issubclass(annotation, StrictRequest)): + offenders.append(f'{app_name} {key}: {getattr(annotation, "__name__", annotation)}') + assert not offenders, ('these twinkle-native routes accept a body that is not a StrictRequest, so an unknown ' + f'field reaches the backend instead of failing: {offenders}') + + +# --------------------------------------------------------------------------- # +# Regression: the sampler routes must bind the sampler-domain models, not model.py's +# +# ``sampler.py`` and ``model.py`` once both declared bare ``AddAdapterRequest`` / +# ``SetTemplateRequest``. Because the handler does ``import twinkle_client.types as +# types`` and the package ``__init__`` re-exported ``model.py`` first, +# ``types.AddAdapterRequest`` resolved to *model.py*'s model -- whose ``config`` is a +# ``str``, so it rejected the dict a real ``add_adapter_to_sampler`` call sends. The +# Sampler-prefixed names remove the collision; this pins the binding so it cannot +# silently regress. +# --------------------------------------------------------------------------- # + + +def _body_model(app, path: str): + from fastapi.routing import APIRoute + for route in app.routes: + if isinstance(route, APIRoute) and route.path == path: + (param, ) = route.dependant.body_params + return param.field_info.annotation + raise AssertionError(f'route {path} not found') + + +def test_sampler_routes_bind_sampler_domain_models(): + from tests.server.contract.client_api_harness import build_sampler_app + from twinkle_client.types import sampler as sampler_types + + app = build_sampler_app() + assert _body_model(app, '/twinkle/add_adapter_to_sampler') is sampler_types.SamplerAddAdapterRequest + assert _body_model(app, '/twinkle/set_template') is sampler_types.SamplerSetTemplateRequest + + +def test_sampler_add_adapter_accepts_a_dict_config_where_model_rejects_it(): + """The exact divergence the collision hid: the client sends ``config`` as a dict.""" + from twinkle_client.types import model as model_types + from twinkle_client.types import sampler as sampler_types + + # The sampler contract (``config: Any``) accepts the LoRA config dict the client sends. + ok = sampler_types.SamplerAddAdapterRequest(adapter_name='a', config={'r': 8}) + assert ok.config == {'r': 8} + # model.py's same-shaped-name model declares ``config: Optional[str]`` and would 422 it, + # which is why the two must not share a bare class name. + with pytest.raises(ValidationError): + model_types.AddAdapterRequest(adapter_name='a', config={'r': 8}) + + +if __name__ == '__main__': + raise SystemExit(pytest.main([__file__, '-v'])) diff --git a/tests/server/validation/test_wire_schema.py b/tests/server/validation/test_wire_schema.py new file mode 100644 index 00000000..69b16f7b --- /dev/null +++ b/tests/server/validation/test_wire_schema.py @@ -0,0 +1,209 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Wire schema behaviour for the inline ``inputs`` data plane. + +These assertions are made against the schema directly, not through a mock backend: the +mock accepts ``**kwargs`` without inspecting anything, so "the mock did not complain" +is evidence of nothing about validation. +""" +from __future__ import annotations + +import pytest +from pydantic import TypeAdapter, ValidationError +from typing import Union, get_args, get_origin + +from twinkle.data_format.encoding import ENCODED_INPUT_KEYS, is_encoded +from twinkle.processor.base import InputProcessor +from twinkle_client.types import data as wire + +_INPUTS = TypeAdapter(wire.WireInputBatch) + + +def _parse(payload): + return _INPUTS.validate_python(payload) + + +# --------------------------------------------------------------------------- # +# Classification and homogeneity +# --------------------------------------------------------------------------- # + + +def test_encoded_entry_parses_as_input_feature(): + (entry, ) = _parse([{'input_ids': [1, 2, 3]}]) + assert isinstance(entry, wire.WireInputFeature) + + +def test_embedding_only_entry_is_encoded_not_a_trajectory(): + """An embedding-only batch is already encoded. + + Reading it as a ``Trajectory`` would send it through ``template.batch_encode`` and + fail far from the cause, which is why ``input_embedding`` is part of the shared + predicate rather than only ``input_ids``. + """ + (entry, ) = _parse([{'input_embedding': [[0.1, 0.2]]}]) + assert isinstance(entry, wire.WireInputFeature) + + +def test_message_entry_parses_as_trajectory(): + (entry, ) = _parse([{'messages': [{'role': 'user', 'content': 'hi'}]}]) + assert isinstance(entry, wire.WireTrajectory) + + +def test_mixed_batch_is_rejected(): + with pytest.raises(ValidationError): + _parse([{'input_ids': [1]}, {'messages': [{'role': 'user', 'content': 'x'}]}]) + + +def test_entry_without_a_required_key_is_rejected(): + with pytest.raises(ValidationError): + _parse([{'labels': [1, 2]}]) + + +@pytest.mark.parametrize('payload', [[1, 2, 3], 'text', 42, True, None]) +def test_non_object_inputs_are_rejected(payload): + with pytest.raises(ValidationError): + _parse(payload) + + +def test_a_single_entry_is_accepted_as_a_one_element_batch(): + assert len(_parse({'input_ids': [1, 2]})) == 1 + + +def test_required_key_rule_matches_the_shared_predicate(): + """The schema's required-key rule and ``is_encoded`` must stay the same rule. + + If they drifted, an entry could be an ``InputFeature`` to one and a ``Trajectory`` + to the other -- the exact divergence the single shared predicate exists to prevent. + """ + for key in ENCODED_INPUT_KEYS: + entry = {key: [1] if key == 'input_ids' else [[0.5]]} + assert is_encoded(entry) + assert isinstance(_parse([entry])[0], wire.WireInputFeature) + assert not is_encoded({'messages': []}) + + +# --------------------------------------------------------------------------- # +# Strictness on declared numeric fields +# --------------------------------------------------------------------------- # + + +def test_bool_tokens_are_rejected(): + """Lax ``int`` would coerce ``[true, false]`` to ``[1, 0]`` and train on it.""" + with pytest.raises(ValidationError): + _parse([{'input_ids': [True, False]}]) + + +def test_float_tokens_are_rejected(): + """Values come from a tensor's ``tolist()``; a ``1.0`` there is an upstream defect.""" + with pytest.raises(ValidationError): + _parse([{'input_ids': [1.0]}]) + + +def test_negative_labels_are_accepted(): + (entry, ) = _parse([{'input_ids': [1, 2], 'labels': [-100, 5]}]) + assert entry.labels == [-100, 5] + + +def test_float_vlm_values_are_accepted(): + (entry, ) = _parse([{'input_ids': [1], 'pixel_values': [[0.5, 0.25]]}]) + assert entry.pixel_values == [[0.5, 0.25]] + + +@pytest.mark.parametrize('position_ids', [[0, 1], [[0, 1], [2, 3]], [[[0, 1]]]]) +def test_position_ids_accept_one_to_three_dimensions(position_ids): + (entry, ) = _parse([{'input_ids': [1, 2], 'position_ids': position_ids}]) + assert entry.position_ids == position_ids + + +def test_position_ids_reject_a_scalar(): + with pytest.raises(ValidationError): + _parse([{'input_ids': [1, 2], 'position_ids': 0}]) + + +def test_routed_experts_require_exactly_three_dimensions(): + with pytest.raises(ValidationError): + _parse([{'input_ids': [1, 2], 'routed_experts': [0, 1]}]) + + +# --------------------------------------------------------------------------- # +# Extension data and export +# --------------------------------------------------------------------------- # + + +def test_unknown_json_fields_survive_a_round_trip(): + """A preprocessor's leftover columns must reach the backend, not be dropped. + + ``extra='ignore'`` would accept the request and then silently strip these on export, + which loses data the caller sent -- a worse outcome than rejecting it. + """ + payload = {'input_ids': [1, 2], 'source_id': 'row-7', 'score': 0.5} + exported = wire.export(_parse([payload])[0]) + assert exported == payload + + +def test_export_omits_unset_optional_fields(): + """Twinkle_Core branches on key *presence*, so ``None`` must not be emitted.""" + exported = wire.export(_parse([{'input_ids': [1, 2]}])[0]) + assert exported == {'input_ids': [1, 2]} + + +def test_round_trip_is_idempotent(): + samples = [ + {'input_ids': [1, 2, 3]}, + {'input_ids': [[1, 2], [3, 4]]}, + {'input_ids': [1, 2], 'position_ids': [[0, 1]]}, + {'input_ids': [1, 2], 'position_ids': [[[0, 1]]]}, + {'input_ids': [1, 2], 'routed_experts': [[[0, 1]]]}, + {'input_ids': [1, 2], 'labels': [-100, 4]}, + {'input_embedding': [[0.5]]}, + {'messages': [{'role': 'user', 'content': 'x'}], 'user_data': [['k', '"v"']]}, + {'messages': []}, + ] + for sample in samples: + once = wire.export(_parse([sample])[0]) + twice = wire.export(_parse([once])[0]) + assert once == twice, sample + + +# --------------------------------------------------------------------------- # +# Structural invariants +# --------------------------------------------------------------------------- # + + +def _depth(annotation) -> int: + depth = 0 + while get_origin(annotation) is list: + depth += 1 + annotation = get_args(annotation)[0] + return depth + + +@pytest.mark.parametrize('alias', ['Ints1to2', 'Ints1to3', 'Numbers1to2', 'Numbers1to4']) +def test_union_members_are_declared_shallowest_first(alias): + """Deepest-first is ~41x slower on a 2-D input, so the order is load-bearing. + + Asserted structurally rather than by timing: a timing assertion would have to build + the anti-pattern to compare against, take seconds, and could fail on a busy runner. + """ + annotation = getattr(wire, alias) + union = get_args(annotation)[0] # unwrap Annotated + assert get_origin(union) is Union + depths = [_depth(member) for member in get_args(union)] + assert depths == sorted(depths) and len(set(depths)) == len(depths), depths + + +def test_schema_covers_every_key_core_reads(): + missing = wire.CORE_INPUT_KEYS - wire.declared_wire_keys() + assert not missing, f'Twinkle_Core reads these keys but the wire schema drops them: {sorted(missing)}' + + +def test_vlm_field_set_matches_the_processor(): + """Kept as a test, not an import, so this module stays free of Twinkle_Core's deps. + + A field added to ``VLM_CONCAT_FIELDS`` and not here would be silently absent from + the wire while the batching code still expects it. + """ + assert wire.VLM_TENSOR_FIELDS == frozenset(InputProcessor.VLM_CONCAT_FIELDS) + + +if __name__ == '__main__': + raise SystemExit(pytest.main([__file__, '-v'])) diff --git a/tests/twinkle_client/test_async_components.py b/tests/twinkle_client/test_async_components.py index 55689637..5dcab1af 100644 --- a/tests/twinkle_client/test_async_components.py +++ b/tests/twinkle_client/test_async_components.py @@ -1,7 +1,16 @@ # Copyright (c) ModelScope Contributors. All rights reserved. +"""Client-side wire shape of the async / data-plane component calls. + +The recorded body is the *serialized request model*, not a hand-built dict: every +twinkle-native client method now instantiates its endpoint's model and posts one +``model_dump_json``. Asserting on the parsed JSON therefore checks the real wire shape, +including the fact that a caller's undeclared arguments land in the model's passthrough +region rather than at the top level. +""" from __future__ import annotations import asyncio +import json from twinkle_client.types import DataRef @@ -11,6 +20,7 @@ class _Response: def __init__(self, payload, status_code: int = 200): self._payload = payload self.status_code = status_code + self.ok = status_code < 400 def raise_for_status(self) -> None: if self.status_code >= 400: @@ -25,20 +35,32 @@ def _completed(result): return {'request_id': 'req-test', 'status': 'completed', 'result': result} -def test_model_forward_backward_sends_multiple_data_refs(monkeypatch) -> None: - import twinkle_client.http as http_module - from twinkle_client.model import multi_lora_transformers as module +def _recorder(calls, result_factory): + """A ``requests.post`` stand-in that records the URL and the decoded JSON body.""" - calls = [] + def post(url, headers=None, data=None, timeout=None, **_kwargs): + body = json.loads(data) if data else {} + calls.append((url, body)) + return _Response(result_factory(url)) - def post(*, url, json_data=None, **_kwargs): - calls.append((url, json_data)) - if url.endswith('/create'): - return _Response({}) - return _Response(_completed({'result': {'loss': 1.0}})) + return post + +def _patch_transport(monkeypatch, calls, result_factory): + import twinkle_client.http.http_utils as http_utils + import twinkle_client.http as http_module monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') - monkeypatch.setattr(module, 'http_post', post) + monkeypatch.setattr(http_utils, 'get_base_url', lambda: 'http://server/api/v1') + monkeypatch.setattr(http_utils, 'get_api_key', lambda: 'test-key') + monkeypatch.setattr(http_utils.requests, 'post', _recorder(calls, result_factory)) + + +def test_model_forward_backward_sends_multiple_data_refs(monkeypatch) -> None: + from twinkle_client.model import multi_lora_transformers as module + + calls: list = [] + _patch_transport(monkeypatch, calls, + lambda url: {} if url.endswith('/create') else _completed({'result': {'loss': 1.0}})) model = module.MultiLoraTransformersModel('ms://base') model.adapter_name = 'adapter' @@ -61,17 +83,10 @@ def post(*, url, json_data=None, **_kwargs): def test_model_inline_forward_methods_keep_the_original_endpoints(monkeypatch) -> None: - import twinkle_client.http as http_module from twinkle_client.model import multi_lora_transformers as module - calls = [] - - def post(*, url, json_data=None, **_kwargs): - calls.append((url, json_data)) - return _Response({} if url.endswith('/create') else _completed({'result': {}})) - - monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') - monkeypatch.setattr(module, 'http_post', post) + calls: list = [] + _patch_transport(monkeypatch, calls, lambda url: {} if url.endswith('/create') else _completed({'result': {}})) model = module.MultiLoraTransformersModel('ms://base') model.adapter_name = 'adapter' @@ -87,22 +102,41 @@ def post(*, url, json_data=None, **_kwargs): ] assert all(body['inputs'] == inputs for _, body in calls[-3:]) assert all('input_refs' not in body for _, body in calls[-3:]) + # Declared backend parameters stay top level; ``exclude_none`` keeps the rest off. + assert calls[-3][1]['return_logits'] is True + assert calls[-2][1]['disable_lora'] is True + assert calls[-1][1]['micro_batch_size'] == 1 -def test_model_data_plane_forward_uses_a_separate_api(monkeypatch) -> None: - import twinkle_client.http as http_module +def test_undeclared_forward_arguments_are_routed_to_loss_kwargs(monkeypatch) -> None: + """A loss input is not a declared field, so it travels in the passthrough region. + + The public signature is unchanged -- callers still pass ``advantages=...`` -- which is + what lets the body be strict without breaking existing scripts. + """ from twinkle_client.model import multi_lora_transformers as module - calls = [] + calls: list = [] + _patch_transport(monkeypatch, calls, lambda url: {} if url.endswith('/create') else _completed({'result': {}})) - def post(url, json_data=None, **_kwargs): - calls.append((url, json_data)) - return _Response({} if url.endswith('/create') else _completed({'result': {'value': 1}})) + model = module.MultiLoraTransformersModel('ms://base') + model.adapter_name = 'adapter' + model.forward_backward([{'input_ids': [1, 2]}], advantages=[0.5], old_logps=[[-1.0, -2.0]]) - monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') - monkeypatch.setattr(module, 'http_post', post) + _, body = calls[-1] + assert body['loss_kwargs'] == {'advantages': [0.5], 'old_logps': [[-1.0, -2.0]]} + assert 'advantages' not in body + + +def test_model_data_plane_forward_uses_a_separate_api(monkeypatch) -> None: + from twinkle_client.model import multi_lora_transformers as module + + calls: list = [] + _patch_transport(monkeypatch, calls, + lambda url: {} if url.endswith('/create') else _completed({'result': {'value': 1}})) model = module.MultiLoraTransformersModel('ms://base') + model.adapter_name = 'adapter' ref = DataRef(ref_id='data-1', size=2, fields=['train_input']) model.forward_from_data_plane(ref, input_field='train_input') @@ -113,21 +147,19 @@ def post(url, json_data=None, **_kwargs): def test_model_data_plane_forward_only_can_append_selected_outputs(monkeypatch) -> None: - import twinkle_client.http as http_module from twinkle_client.model import multi_lora_transformers as module ref = DataRef(ref_id='data-1', size=2, fields=['input_ids']) updated_ref = ref.model_copy(update={'fields': ['input_ids', 'ref_logps']}) - calls = [] - - def post(*, url, json_data=None, **_kwargs): - calls.append((url, json_data)) - return _Response({} if url.endswith('/create') else _completed({'result': updated_ref.model_dump()})) - - monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') - monkeypatch.setattr(module, 'http_post', post) + calls: list = [] + # The handler wraps its payload as ``{'result': ...}``, so the stub must too -- + # otherwise the test asserts against a reply shape the server never sends. + _patch_transport(monkeypatch, calls, + lambda url: {} if url.endswith('/create') else _completed( + {'result': updated_ref.model_dump()})) model = module.MultiLoraTransformersModel('ms://base') + model.adapter_name = 'adapter' result = model.forward_only_from_data_plane( ref, output_ref=ref, @@ -143,7 +175,6 @@ def post(*, url, json_data=None, **_kwargs): def test_sampler_async_data_plane_path_returns_reference_without_materializing(monkeypatch) -> None: - import twinkle_client.http as http_module from twinkle_client.sampler import vllm_sampler as module output_ref = DataRef( @@ -152,16 +183,10 @@ def test_sampler_async_data_plane_path_returns_reference_without_materializing(m fields=['train_input', 'sampled_logprobs', 'decoded'], kind='rollout', ) - calls = [] + calls: list = [] + _patch_transport(monkeypatch, calls, + lambda url: {} if url.endswith('/create') else _completed(output_ref.model_dump())) - def post(*, url, json_data=None, **_kwargs): - calls.append((url, json_data)) - if url.endswith('/create'): - return _Response({}) - return _Response(_completed(output_ref.model_dump())) - - monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') - monkeypatch.setattr(module, 'http_post', post) sampler = module.vLLMSampler('ms://base') result = asyncio.run(sampler.asample_to_data_plane( diff --git a/tests/twinkle_client/test_request_builder.py b/tests/twinkle_client/test_request_builder.py new file mode 100644 index 00000000..c65d9d1d --- /dev/null +++ b/tests/twinkle_client/test_request_builder.py @@ -0,0 +1,156 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Level 0: the client builds the request from the schema, in-process. + +The property worth testing is that a bad call fails *without a network round trip*. Every +test here monkeypatches ``requests.post`` to fail loudly, so any test that passes has +proved no request was sent. +""" +from __future__ import annotations + +import pytest +from pydantic import JsonValue, ValidationError +from typing import Dict + +from twinkle_client._request_builder import build_request, request_json, to_wire_value +from twinkle_client.exceptions import TwinkleClientValidationError +from twinkle_client.types import model as model_types +from twinkle_client.types.base import StrictRequest, passthrough + + +@pytest.fixture(autouse=True) +def no_network(monkeypatch): + """Any HTTP call in this module is a bug in the code under test.""" + import twinkle_client.http.http_utils as http_utils + monkeypatch.setattr(http_utils.requests, 'post', + lambda *a, **k: pytest.fail('a Level 0 failure must not produce a request')) + + +# --------------------------------------------------------------------------- # +# Routing +# --------------------------------------------------------------------------- # + + +def test_a_declared_name_goes_to_its_field(): + body = build_request(model_types.ForwardRequest, inputs=[{'input_ids': [1]}], adapter_name='a', task='embedding') + assert body.task == 'embedding' + assert body.loss_kwargs == {} + + +def test_an_undeclared_name_goes_to_the_single_passthrough_region(): + """Public signatures stay ``**kwargs``; only the wire shape changes.""" + body = build_request( + model_types.ForwardBackwardTaskRequest, + inputs=[{'input_ids': [1]}], + adapter_name='a', + advantages=[0.5], + old_logps=[[-1.0]]) + assert body.loss_kwargs == {'advantages': [0.5], 'old_logps': [[-1.0]]} + + +def test_an_ambiguous_target_is_an_error_rather_than_a_guess(): + """With two regions the builder refuses to choose instead of guessing. + + A model that grows a second region -- constructor arguments *and* invoked-method + arguments, say -- has no name-based rule that can tell them apart, and guessing wrong + sends a valid argument to the wrong callable: a wrong result rather than an error. No + shipped model has two today; the rule exists so that adding one fails loudly. + """ + + class _TwoRegions(StrictRequest): + target: str + init_kwargs: Dict[str, JsonValue] = passthrough() + call_kwargs: Dict[str, JsonValue] = passthrough() + + with pytest.raises(TwinkleClientValidationError) as raised: + build_request(_TwoRegions, target='t', unplaceable=1) + assert 'init_kwargs' in str(raised.value) and 'call_kwargs' in str(raised.value) + + +def test_a_model_without_a_region_rejects_an_unknown_name(): + with pytest.raises(TwinkleClientValidationError) as raised: + build_request(model_types.SaveRequest, adapter_name='a', name='ckpt', typo=1) + assert 'typo' in str(raised.value) + + +def test_an_explicit_region_and_a_routed_key_are_merged(): + body = build_request( + model_types.SetLossRequest, loss_cls='DPOLoss', adapter_name='a', init_kwargs={'beta': 0.1}, loss_type='sigmoid') + assert body.init_kwargs == {'beta': 0.1, 'loss_type': 'sigmoid'} + + +def test_a_key_passed_twice_is_an_error(): + """Silently letting one win would make the effective value depend on merge order.""" + with pytest.raises(TwinkleClientValidationError, match='both directly and inside'): + build_request( + model_types.SetLossRequest, loss_cls='DPOLoss', adapter_name='a', init_kwargs={'beta': 0.1}, beta=0.2) + + +def test_an_omitted_optional_argument_is_not_routed_into_the_region(): + """Client methods pass optionals unconditionally. + + Routing an explicit ``None`` would hand the backend a null argument it never received + before, changing behaviour for callers who simply did not pass anything. + """ + body = build_request(model_types.SetLossRequest, loss_cls='DPOLoss', adapter_name='a', unused=None) + assert body.init_kwargs == {} + + +# --------------------------------------------------------------------------- # +# Validation before the wire +# --------------------------------------------------------------------------- # + + +def test_a_wrongly_typed_field_fails_in_process(): + with pytest.raises(ValidationError): + build_request(model_types.ForwardRequest, inputs=[{'input_ids': [1]}], adapter_name='a', temperature='hot') + + +def test_an_out_of_range_value_fails_in_process(): + with pytest.raises(ValidationError): + build_request(model_types.ForwardRequest, inputs=[{'input_ids': [1]}], adapter_name='a', temperature=0) + + +def test_malformed_inputs_fail_in_process(): + """The client shares the server's schema, so it catches this without asking.""" + with pytest.raises(ValidationError): + build_request(model_types.ForwardRequest, inputs=[{'input_ids': [True]}], adapter_name='a') + + +def test_a_missing_required_field_fails_in_process(): + with pytest.raises(ValidationError): + build_request(model_types.ForwardRequest, inputs=[{'input_ids': [1]}]) + + +# --------------------------------------------------------------------------- # +# Serialization +# --------------------------------------------------------------------------- # + + +def test_unset_optionals_stay_off_the_wire(): + """Absent and "not requested" must look the same, or the server needs its own defaults.""" + import json + body = build_request(model_types.ForwardRequest, inputs=[{'input_ids': [1]}], adapter_name='a') + payload = json.loads(request_json(body)) + assert payload == {'inputs': [{'input_ids': [1]}], 'adapter_name': 'a', 'loss_kwargs': {}} + + +def test_a_lora_config_is_serialized_to_the_form_the_server_decodes(): + from peft import LoraConfig + wire = to_wire_value(LoraConfig(target_modules='all-linear')) + assert isinstance(wire, str) and 'LoraConfig' in wire + + +def test_a_component_handle_is_sent_as_its_id(): + class _Handle: + processor_id = 'pid:abc' + + assert to_wire_value(_Handle()) == 'pid:abc' + + +def test_numpy_values_are_converted_to_lists(): + import numpy as np + assert to_wire_value(np.array([1, 2])) == [1, 2] + + +if __name__ == '__main__': + raise SystemExit(pytest.main([__file__, '-v'])) From 4f1addb7991767348be27007b145a4539c39e45e Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Fri, 18 Sep 2026 06:44:35 +0000 Subject: [PATCH 19/32] style(server): satisfy pre-commit lint on request-schema Auto-fixes flagged by CI on b928a171 (pre-commit run --all-files): - sampler/twinkle_handlers.py: isort import order + wrap the 122-char create() signature (E501). - validation/backend_compat.py: pyupgrade Optional[str] -> str | None. --- src/twinkle/server/sampler/twinkle_handlers.py | 5 +++-- src/twinkle/server/validation/backend_compat.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/twinkle/server/sampler/twinkle_handlers.py b/src/twinkle/server/sampler/twinkle_handlers.py index c50e9961..a0704d3f 100644 --- a/src/twinkle/server/sampler/twinkle_handlers.py +++ b/src/twinkle/server/sampler/twinkle_handlers.py @@ -21,7 +21,6 @@ import numpy as np import twinkle_client.types as types -from twinkle_client.types import sampler as sampler_types from twinkle.data_format import SamplingParams from twinkle.server.exceptions import RequestRejectedError from twinkle.server.lifecycle.submit import backend_kwargs, resolve_twinkle_adapter_name, to_backend_inputs @@ -31,6 +30,7 @@ from twinkle.server.utils.task_errors import task_error_payload from twinkle.utils.logger import get_logger from twinkle_client.common.json_utils import json_safe +from twinkle_client.types import sampler as sampler_types logger = get_logger() @@ -216,7 +216,8 @@ def _register_twinkle_sampler_routes(app: FastAPI, self_fn: Callable[[], Sampler """ @app.post('/twinkle/create', response_model=sampler_types.SamplerCreateResponse) - async def create(request: Request, self: SamplerManagement = Depends(self_fn)) -> sampler_types.SamplerCreateResponse: + async def create( + request: Request, self: SamplerManagement = Depends(self_fn)) -> sampler_types.SamplerCreateResponse: """Health check / session creation endpoint.""" return sampler_types.SamplerCreateResponse() diff --git a/src/twinkle/server/validation/backend_compat.py b/src/twinkle/server/validation/backend_compat.py index 1986a9df..6587ab4b 100644 --- a/src/twinkle/server/validation/backend_compat.py +++ b/src/twinkle/server/validation/backend_compat.py @@ -68,7 +68,7 @@ class EndpointUnavailableError(TwinkleServerError): category = ErrorCategory.Server -def resolve_backend(service: Any) -> Optional[str]: +def resolve_backend(service: Any) -> str | None: """This deployment's declared backend, or ``None`` when it has no backend concept. Read from the deployment's own configuration (``ModelManagement.backend``), never @@ -80,7 +80,7 @@ def resolve_backend(service: Any) -> Optional[str]: return backend if isinstance(backend, str) else None -def assert_endpoint_available(service: Any, capability: Optional[str]) -> None: +def assert_endpoint_available(service: Any, capability: str | None) -> None: """Raise 501 when this deployment's backend does not implement ``capability``.""" if capability is None: return @@ -113,7 +113,7 @@ def assert_backend_fields(service: Any, body: Any) -> None: error_code=422) -def assert_request_supported(service: Any, body: Any, *, capability: Optional[str] = None) -> None: +def assert_request_supported(service: Any, body: Any, *, capability: str | None = None) -> None: """The single preflight entry point, called from ``run_submit`` before the enqueue. Both checks read *declared* metadata, so neither can produce a false positive. A From 637c53d2c1c6290ecce5a0ce6147d3eea43fc2f4 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Fri, 18 Sep 2026 07:21:04 +0000 Subject: [PATCH 20/32] style(client): enforce lint on src/twinkle_client and normalize existing code Remove src/twinkle_client from the pre-commit exemptions (flake8/isort/yapf/ pyupgrade + the whitespace/EOL/quote fixers) so the client is linted like the server, and bring the existing (previously-unchecked) client code to a green `pre-commit run --all-files`: - isort/yapf/pyupgrade/double-quote normalization across the client package. - Wrap over-length tool/description/error strings (implicit concatenation, content preserved) in auto/agent/tools.py and utils/patch_tinker.py. - Rename ambiguous loop var `l` -> `line` (E741) in auto/agent/monitor.py. - `# noqa: E402` on the intentional late import in twinkle_client/__init__.py. - setup.cfg per-file-ignores: E501 for auto/agent/monitor.py (embedded LLM prompt with verbatim long lines). - types/__init__.py: scoped `# yapf: disable` around the re-export block so the 45-name `from .model import (...)` stops oscillating between isort's aligned wrap and yapf's hanging wrap (isort still owns the ordering). --- .pre-commit-config.yaml | 22 +- setup.cfg | 3 + src/twinkle_client/__init__.py | 11 +- src/twinkle_client/_future.py | 13 +- src/twinkle_client/_request_builder.py | 3 +- src/twinkle_client/async_rl/workers.py | 5 +- src/twinkle_client/auto/__init__.py | 45 +- src/twinkle_client/auto/agent/core.py | 17 +- src/twinkle_client/auto/agent/monitor.py | 20 +- src/twinkle_client/auto/agent/tools.py | 480 +++++++++++------- src/twinkle_client/auto/app.py | 10 +- src/twinkle_client/auto/connection.py | 9 +- src/twinkle_client/auto/runtime.py | 14 +- src/twinkle_client/common/component_rpc.py | 3 +- src/twinkle_client/common/json_utils.py | 4 +- src/twinkle_client/data_plane.py | 18 +- src/twinkle_client/dataloader/dataloader.py | 2 +- src/twinkle_client/dataset/base.py | 2 +- src/twinkle_client/exceptions.py | 15 +- src/twinkle_client/http/__init__.py | 4 +- src/twinkle_client/http/http_utils.py | 6 +- src/twinkle_client/manager.py | 32 +- .../model/multi_lora_transformers.py | 11 +- src/twinkle_client/processor/base.py | 2 +- src/twinkle_client/rollout/multi_turn.py | 29 +- src/twinkle_client/sampler/vllm_sampler.py | 22 +- src/twinkle_client/skills/base.py | 3 +- src/twinkle_client/skills/manager.py | 7 +- .../skills/modelscope_provider.py | 18 +- src/twinkle_client/types/__init__.py | 156 ++---- src/twinkle_client/types/base.py | 2 +- src/twinkle_client/types/component.py | 20 +- src/twinkle_client/types/data.py | 70 +-- src/twinkle_client/types/errors.py | 6 +- src/twinkle_client/types/lifecycle.py | 8 +- src/twinkle_client/types/model.py | 112 ++-- src/twinkle_client/types/processor.py | 4 +- src/twinkle_client/types/sampler.py | 27 +- src/twinkle_client/utils/patch_tinker.py | 11 +- 39 files changed, 626 insertions(+), 620 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d254bd53..52548fad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,42 +3,42 @@ repos: rev: 7.3.0 hooks: - id: flake8 - exclude: ^(examples/|cookbook/|client_tools/|src/twinkle_client/|tests/) + exclude: ^(examples/|cookbook/|client_tools/|tests/) - repo: https://github.com/PyCQA/isort rev: 7.0.0 hooks: - id: isort - exclude: ^(examples/|cookbook/|client_tools/|src/twinkle_client/|tests/) + exclude: ^(examples/|cookbook/|client_tools/|tests/) - repo: https://github.com/google/yapf rev: v0.43.0 hooks: - id: yapf - exclude: ^(examples/|cookbook/|client_tools/|src/twinkle_client/|tests/) + exclude: ^(examples/|cookbook/|client_tools/|tests/) - repo: https://github.com/asottile/pyupgrade rev: v3.19.1 hooks: - id: pyupgrade args: [--py38-plus] - exclude: ^(examples/|cookbook/|client_tools/|src/twinkle_client/|tests/) + exclude: ^(examples/|cookbook/|client_tools/|tests/) - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: trailing-whitespace - exclude: ^(client_tools/|src/twinkle_client/|tests/) + exclude: ^(client_tools/|tests/) - id: check-yaml - exclude: ^(client_tools/|src/twinkle_client/|tests/) + exclude: ^(client_tools/|tests/) - id: end-of-file-fixer - exclude: ^(client_tools/|src/twinkle_client/|tests/) + exclude: ^(client_tools/|tests/) - id: requirements-txt-fixer - exclude: ^(client_tools/|src/twinkle_client/|tests/) + exclude: ^(client_tools/|tests/) - id: double-quote-string-fixer - exclude: ^(client_tools/|src/twinkle_client/|tests/) + exclude: ^(client_tools/|tests/) - id: check-merge-conflict - exclude: ^(client_tools/|src/twinkle_client/|tests/) + exclude: ^(client_tools/|tests/) - id: mixed-line-ending args: ["--fix=lf"] - exclude: ^(client_tools/|src/twinkle_client/|tests/) + exclude: ^(client_tools/|tests/) diff --git a/setup.cfg b/setup.cfg index 811fd55c..0454a844 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,9 @@ max-line-length = 120 select = B,E,F,P,T4,W,B9 ignore = F401,F403,F405,F821,W503,E251,W504,E126,E125 exclude = docs/src,*.pyi,.git,peft.py +per-file-ignores = + # Embedded LLM prompt/tool text with intentionally long lines that must stay verbatim. + src/twinkle_client/auto/agent/monitor.py:E501 [darglint] ignore=DAR101 diff --git a/src/twinkle_client/__init__.py b/src/twinkle_client/__init__.py index bb13f19a..8dc72a31 100644 --- a/src/twinkle_client/__init__.py +++ b/src/twinkle_client/__init__.py @@ -1,6 +1,7 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from __future__ import annotations -from typing import Optional, TYPE_CHECKING + +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from .manager import TwinkleClient @@ -35,11 +36,11 @@ def init_tinker_client(**kwargs) -> None: def init_twinkle_client( - base_url: Optional[str] = None, - api_key: Optional[str] = None, + base_url: str | None = None, + api_key: str | None = None, session_heartbeat_interval: int = 10, **kwargs, -) -> 'TwinkleClient': +) -> TwinkleClient: """ Initialize a Twinkle client. @@ -72,6 +73,6 @@ def init_twinkle_client( ) -from .data_plane import DataPlaneClient +from .data_plane import DataPlaneClient # noqa: E402 (late import: keeps optional data-plane dep off the hot path) __all__ = ['DataPlaneClient', 'init_tinker_client', 'init_twinkle_client'] diff --git a/src/twinkle_client/_future.py b/src/twinkle_client/_future.py index 65a2343e..318a3ac2 100644 --- a/src/twinkle_client/_future.py +++ b/src/twinkle_client/_future.py @@ -8,15 +8,14 @@ from __future__ import annotations import logging +import requests import time from typing import Any, Optional -import requests - from twinkle_client.exceptions import TaskCancelledError, TaskFailedError, TaskRecordLostError, TaskWaitTimeoutError from twinkle_client.http import http_post from twinkle_client.http.utils import get_base_url -from twinkle_client.types.lifecycle import TaskEnvelope, TERMINAL_STATUSES +from twinkle_client.types.lifecycle import TERMINAL_STATUSES, TaskEnvelope logger = logging.getLogger('twinkle_client') @@ -65,7 +64,7 @@ def _post_retrieve(request_id: str) -> TaskEnvelope: return TaskEnvelope.model_validate(response.json()) -def _status_of(error: requests.HTTPError) -> Optional[int]: +def _status_of(error: requests.HTTPError) -> int | None: status = getattr(error, 'status_code', None) if status is None and getattr(error, 'response', None) is not None: status = error.response.status_code @@ -123,7 +122,7 @@ def resolve(submit: TaskEnvelope, *, model_cls, total_timeout: float = _DEFAULT_ long-poll. Only transport retries back off. """ if submit.status in TERMINAL_STATUSES: - return _unwrap(submit, model_cls) # same call as the retrieve path + return _unwrap(submit, model_cls) # same call as the retrieve path deadline = time.monotonic() + total_timeout transport_failures = not_found_count = 0 @@ -146,10 +145,10 @@ def resolve(submit: TaskEnvelope, *, model_cls, total_timeout: float = _DEFAULT_ transport_failures += 1 if transport_failures > _TRANSPORT_RETRY_MAX: raise - time.sleep(min(2 ** transport_failures, 30)) + time.sleep(min(2**transport_failures, 30)) continue if reply.status in TERMINAL_STATUSES: - return _unwrap(reply, model_cls) # same call as the submit path + return _unwrap(reply, model_cls) # same call as the submit path _log_queue_state(reply) except (KeyboardInterrupt, SystemExit): # Caller abandoned the wait: best-effort ask the server to drop the task if it diff --git a/src/twinkle_client/_request_builder.py b/src/twinkle_client/_request_builder.py index 1396cac4..6b1e4bfd 100644 --- a/src/twinkle_client/_request_builder.py +++ b/src/twinkle_client/_request_builder.py @@ -42,8 +42,9 @@ def to_wire_value(value: Any) -> Any: """ if hasattr(value, 'processor_id'): return value.processor_id - from twinkle.dataset import DatasetMeta from peft import LoraConfig + + from twinkle.dataset import DatasetMeta if isinstance(value, (DatasetMeta, LoraConfig)): from twinkle_client.common.serialize import serialize_object return serialize_object(value) diff --git a/src/twinkle_client/async_rl/workers.py b/src/twinkle_client/async_rl/workers.py index 8d105bcf..a9014fd0 100644 --- a/src/twinkle_client/async_rl/workers.py +++ b/src/twinkle_client/async_rl/workers.py @@ -37,10 +37,7 @@ def __init__(self, workers: Sequence[Worker]) -> None: raise ValueError(f'worker names must be unique, got {names}') async def run(self) -> None: - tasks = { - asyncio.create_task(worker.run(), name=worker.name): worker - for worker in self.workers - } + tasks = {asyncio.create_task(worker.run(), name=worker.name): worker for worker in self.workers} try: done, pending = await asyncio.wait(tasks, return_when=asyncio.FIRST_EXCEPTION) failure = next( diff --git a/src/twinkle_client/auto/__init__.py b/src/twinkle_client/auto/__init__.py index 3916c91e..27a42cc6 100644 --- a/src/twinkle_client/auto/__init__.py +++ b/src/twinkle_client/auto/__init__.py @@ -18,12 +18,16 @@ def _configure_logging(verbose: bool = False) -> None: from logging.handlers import RotatingFileHandler handler = RotatingFileHandler( - _LOG_FILE, maxBytes=5 * 1024 * 1024, backupCount=3, encoding='utf-8', + _LOG_FILE, + maxBytes=5 * 1024 * 1024, + backupCount=3, + encoding='utf-8', ) - handler.setFormatter(logging.Formatter( - '%(asctime)s [%(levelname)s] %(name)s: %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - )) + handler.setFormatter( + logging.Formatter( + '%(asctime)s [%(levelname)s] %(name)s: %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + )) level = logging.DEBUG if verbose else logging.INFO twinkle_logger = logging.getLogger('twinkle') @@ -45,6 +49,7 @@ def main(argv: list[str] | None = None) -> int: """ import sys import typer + from twinkle.version import __version__ app = typer.Typer( @@ -61,43 +66,51 @@ def _version_callback(value: bool) -> None: @app.command() def launch( run_id: str | None = typer.Option( - None, '--run-id', '-r', + None, + '--run-id', + '-r', envvar='TWINKLE_AUTO_RUN_ID', help='Attach to an existing training run by ID.', ), llm_base_url: str = typer.Option( - 'http://localhost:11434/v1', '--llm-base-url', + 'http://localhost:11434/v1', + '--llm-base-url', envvar='TWINKLE_LLM_BASE_URL', help='LLM API base URL.', ), llm_model: str = typer.Option( - 'qwen3.5', '--llm-model', + 'qwen3.5', + '--llm-model', envvar='TWINKLE_LLM_MODEL', help='LLM model name.', ), llm_api_key: str = typer.Option( - 'not-needed', '--llm-api-key', + 'not-needed', + '--llm-api-key', envvar='TWINKLE_LLM_API_KEY', help='LLM API key.', ), verbose: bool = typer.Option( - False, '--verbose', '-v', + False, + '--verbose', + '-v', envvar='TWINKLE_AUTO_VERBOSE', help='Enable verbose (DEBUG) logging.', ), version: bool = typer.Option( - False, '--version', '-V', - callback=_version_callback, is_eager=True, + False, + '--version', + '-V', + callback=_version_callback, + is_eager=True, help='Show version and exit.', ), ) -> None: """Launch Twinkle Auto.""" _configure_logging(verbose=verbose) logger = get_logger() - logger.info( - f'Auto starting — model={llm_model}, base_url={llm_base_url}, ' - f'run_id={run_id}, log_file={_LOG_FILE}' - ) + logger.info(f'Auto starting — model={llm_model}, base_url={llm_base_url}, ' + f'run_id={run_id}, log_file={_LOG_FILE}') from twinkle_client.auto.app import TwinkleAuto diff --git a/src/twinkle_client/auto/agent/core.py b/src/twinkle_client/auto/agent/core.py index a3c3c5b1..d2c4056c 100644 --- a/src/twinkle_client/auto/agent/core.py +++ b/src/twinkle_client/auto/agent/core.py @@ -5,9 +5,9 @@ import asyncio import json -from twinkle.utils.logger import get_logger from typing import Any, Callable +from twinkle.utils.logger import get_logger from twinkle_client.auto.agent.prompts import SYSTEM_PROMPT from twinkle_client.auto.agent.tools import TOOL_SCHEMAS, ToolExecutor from twinkle_client.auto.connection import LocalConnection @@ -28,7 +28,7 @@ class AgentLoop: def __init__( self, connection: LocalConnection, - llm_client: 'AsyncOpenAI', + llm_client: AsyncOpenAI, llm_model: str, skills_prompt: str = '', ): @@ -41,7 +41,10 @@ def __init__( if skills_prompt: full_prompt = f'{SYSTEM_PROMPT}\n\n{skills_prompt}' self.history: list[dict[str, Any]] = [ - {'role': 'system', 'content': full_prompt}, + { + 'role': 'system', + 'content': full_prompt + }, ] async def send( @@ -94,7 +97,8 @@ async def send( except json.JSONDecodeError as e: logger.error(f'Tool {func_name}: invalid JSON args: {e}\n raw={raw_args[:500]}') args = {} - logger.info(f'Executing tool: {func_name}({", ".join(f"{k}={v!r}" for k, v in list(args.items())[:5])})') + logger.info( + f'Executing tool: {func_name}({", ".join(f"{k}={v!r}" for k, v in list(args.items())[:5])})') result = await self._tool_executor.execute(func_name, args) logger.debug(f'Tool {func_name} result ({len(result)} chars): {result[:300]}') self.history.append({ @@ -156,7 +160,10 @@ async def _call_llm_stream( tool_calls_map[idx] = { 'id': '', 'type': 'function', - 'function': {'name': '', 'arguments': ''}, + 'function': { + 'name': '', + 'arguments': '' + }, } tc = tool_calls_map[idx] if tc_delta.id: diff --git a/src/twinkle_client/auto/agent/monitor.py b/src/twinkle_client/auto/agent/monitor.py index b25169f0..1987f076 100644 --- a/src/twinkle_client/auto/agent/monitor.py +++ b/src/twinkle_client/auto/agent/monitor.py @@ -95,7 +95,7 @@ def __init__( self, connection: LocalConnection, on_message: Callable[[str], None], - llm_client: 'AsyncOpenAI', + llm_client: AsyncOpenAI, llm_model: str = 'qwen3.5', poll_interval: float = 30.0, ): @@ -305,8 +305,14 @@ async def _ask_llm(self, snapshot: dict[str, Any]) -> str | None: response = await self._client.chat.completions.create( model=self.llm_model, messages=[ - {'role': 'system', 'content': MONITOR_SYSTEM_PROMPT + extra}, - {'role': 'user', 'content': user_content}, + { + 'role': 'system', + 'content': MONITOR_SYSTEM_PROMPT + extra + }, + { + 'role': 'user', + 'content': user_content + }, ], temperature=0.3, max_tokens=4096, @@ -340,10 +346,8 @@ async def _apply_fix(self, run_id: str, diagnosis: str, fixed_script: str) -> No """Apply auto-fix: update script + resume training.""" attempts = self._fix_attempts.get(run_id, 0) if attempts >= _MAX_FIX_ATTEMPTS: - self.on_message( - f'[Monitor] 已达最大自动修复次数 ({_MAX_FIX_ATTEMPTS})，不再尝试。' - '请手动检查或输入指令。' - ) + self.on_message(f'[Monitor] 已达最大自动修复次数 ({_MAX_FIX_ATTEMPTS})，不再尝试。' + '请手动检查或输入指令。') return self.on_message(f'[Monitor] 检测到问题，正在自动修复 (第{attempts + 1}次)...\n诊断: {diagnosis}') @@ -390,7 +394,7 @@ def _parse_fix_response(response: str) -> tuple[str, str]: else: # Fallback: text before the python block before = response[:response.find('```python')] - lines = [l.strip() for l in before.splitlines() if l.strip() and not l.startswith('```')] + lines = [line.strip() for line in before.splitlines() if line.strip() and not line.startswith('```')] diagnosis = lines[-1] if lines else 'Auto-fix applied' return diagnosis, fixed_script diff --git a/src/twinkle_client/auto/agent/tools.py b/src/twinkle_client/auto/agent/tools.py index 3689d59e..9a3c7388 100644 --- a/src/twinkle_client/auto/agent/tools.py +++ b/src/twinkle_client/auto/agent/tools.py @@ -23,7 +23,11 @@ 'function': { 'name': 'list_training_runs', 'description': 'List all active and historical training runs.', - 'parameters': {'type': 'object', 'properties': {}, 'required': []}, + 'parameters': { + 'type': 'object', + 'properties': {}, + 'required': [] + }, }, }, { @@ -34,7 +38,10 @@ 'parameters': { 'type': 'object', 'properties': { - 'run_id': {'type': 'string', 'description': 'Training run ID.'}, + 'run_id': { + 'type': 'string', + 'description': 'Training run ID.' + }, }, 'required': ['run_id'], }, @@ -43,13 +50,12 @@ { 'type': 'function', 'function': { - 'name': 'start_server', - 'description': ( - 'Start Ray cluster and Twinkle Server. MUST be called before start_training. ' - 'Idempotent: skips if server is already reachable. ' - 'Supports multi-model deployments: one training model + N sampler/teacher models. ' - 'Automatically generates server_config.yaml from parameters.' - ), + 'name': + 'start_server', + 'description': ('Start Ray cluster and Twinkle Server. MUST be called before start_training. ' + 'Idempotent: skips if server is already reachable. ' + 'Supports multi-model deployments: one training model + N sampler/teacher models. ' + 'Automatically generates server_config.yaml from parameters.'), 'parameters': { 'type': 'object', 'properties': { @@ -67,11 +73,10 @@ 'description': 'Training model backend. Default: transformers.', }, 'samplers': { - 'type': 'array', - 'description': ( - 'List of sampler/teacher models for RL/OPD. Each entry deploys ' - 'an inference service (vLLM or torch). Omit for simple SFT.' - ), + 'type': + 'array', + 'description': ('List of sampler/teacher models for RL/OPD. Each entry deploys ' + 'an inference service (vLLM or torch). Omit for simple SFT.'), 'items': { 'type': 'object', 'properties': { @@ -81,21 +86,21 @@ }, 'gpus': { 'type': 'integer', - 'description': 'Total number of GPUs for this sampler. Default: 1. Must equal tp * dp.', + 'description': + 'Total number of GPUs for this sampler. Default: 1. Must equal tp * dp.', }, 'tp': { - 'type': 'integer', - 'description': ( - 'Tensor parallelism size (GPUs per vLLM worker process). ' - 'Use tp>1 for large models that do not fit on a single GPU. Default: 1.' - ), + 'type': + 'integer', + 'description': + ('Tensor parallelism size (GPUs per vLLM worker process). ' + 'Use tp>1 for large models that do not fit on a single GPU. Default: 1.'), }, 'dp': { - 'type': 'integer', - 'description': ( - 'Data parallelism size (number of independent inference replicas). ' - 'If not specified, computed as gpus // tp. Default: 1.' - ), + 'type': + 'integer', + 'description': ('Data parallelism size (number of independent inference replicas). ' + 'If not specified, computed as gpus // tp. Default: 1.'), }, 'engine': { 'type': 'string', @@ -122,32 +127,43 @@ { 'type': 'function', 'function': { - 'name': 'shutdown_server', - 'description': ( - 'Shut down Twinkle Server and Ray cluster. WARNING: This releases all GPU resources ' - 'and DESTROYS model state held in server memory. Only call when training is truly ' - 'finished and you no longer need the server. Model weights/optimizer state in GPU ' - 'will be LOST unless a checkpoint was explicitly saved.' - ), - 'parameters': {'type': 'object', 'properties': {}, 'required': []}, + 'name': + 'shutdown_server', + 'description': ('Shut down Twinkle Server and Ray cluster. WARNING: This releases all GPU resources ' + 'and DESTROYS model state held in server memory. Only call when training is truly ' + 'finished and you no longer need the server. Model weights/optimizer state in GPU ' + 'will be LOST unless a checkpoint was explicitly saved.'), + 'parameters': { + 'type': 'object', + 'properties': {}, + 'required': [] + }, }, }, { 'type': 'function', 'function': { - 'name': 'start_training', - 'description': ( - 'Create a new training run: write the client script, launch it, and start monitoring. ' - 'REQUIRES: Twinkle Server must be running (call start_server first). ' - 'The client script connects to the server — server holds model state in GPU memory. ' - 'Kill client = pause (state preserved). Re-launch client = resume.' - ), + 'name': + 'start_training', + 'description': ('Create a new training run: write the client script, launch it, and start monitoring. ' + 'REQUIRES: Twinkle Server must be running (call start_server first). ' + 'The client script connects to the server — server holds model state in GPU memory. ' + 'Kill client = pause (state preserved). Re-launch client = resume.'), 'parameters': { 'type': 'object', 'properties': { - 'run_id': {'type': 'string', 'description': 'Unique run ID (e.g., "grpo-gsm8k").'}, - 'script_content': {'type': 'string', 'description': 'Full Python source code of the training script.'}, - 'model_id': {'type': 'string', 'description': 'Model identifier for metadata (e.g., "Qwen/Qwen3.5-4B").'}, + 'run_id': { + 'type': 'string', + 'description': 'Unique run ID (e.g., "grpo-gsm8k").' + }, + 'script_content': { + 'type': 'string', + 'description': 'Full Python source code of the training script.' + }, + 'model_id': { + 'type': 'string', + 'description': 'Model identifier for metadata (e.g., "Qwen/Qwen3.5-4B").' + }, }, 'required': ['run_id', 'script_content'], }, @@ -161,7 +177,10 @@ 'parameters': { 'type': 'object', 'properties': { - 'run_id': {'type': 'string', 'description': 'Training run ID to monitor.'}, + 'run_id': { + 'type': 'string', + 'description': 'Training run ID to monitor.' + }, }, 'required': ['run_id'], }, @@ -171,11 +190,15 @@ 'type': 'function', 'function': { 'name': 'pause_training', - 'description': 'Pause training by killing the client process (SIGKILL). Server retains all state — call resume_training to continue.', + 'description': 'Pause training by killing the client process (SIGKILL). Server retains all state — call ' + 'resume_training to continue.', 'parameters': { 'type': 'object', 'properties': { - 'run_id': {'type': 'string', 'description': 'Training run ID to pause.'}, + 'run_id': { + 'type': 'string', + 'description': 'Training run ID to pause.' + }, }, 'required': ['run_id'], }, @@ -189,7 +212,10 @@ 'parameters': { 'type': 'object', 'properties': { - 'run_id': {'type': 'string', 'description': 'Training run ID to resume.'}, + 'run_id': { + 'type': 'string', + 'description': 'Training run ID to resume.' + }, }, 'required': ['run_id'], }, @@ -198,17 +224,19 @@ { 'type': 'function', 'function': { - 'name': 'stop_training', - 'description': ( - 'Gracefully stop the training client (SIGTERM). The script saves a checkpoint ' - 'before exiting. Server retains model/optimizer state in GPU memory — ' - 'use resume_training to continue. Similar to pause_training but with checkpoint save. ' - 'To fully release GPU resources, use shutdown_server.' - ), + 'name': + 'stop_training', + 'description': ('Gracefully stop the training client (SIGTERM). The script saves a checkpoint ' + 'before exiting. Server retains model/optimizer state in GPU memory — ' + 'use resume_training to continue. Similar to pause_training but with checkpoint save. ' + 'To fully release GPU resources, use shutdown_server.'), 'parameters': { 'type': 'object', 'properties': { - 'run_id': {'type': 'string', 'description': 'Training run ID to stop.'}, + 'run_id': { + 'type': 'string', + 'description': 'Training run ID to stop.' + }, }, 'required': ['run_id'], }, @@ -218,12 +246,19 @@ 'type': 'function', 'function': { 'name': 'update_script', - 'description': 'Update the training script for a run. Archives the current train.py as train_v{N}.py and writes the new version. Use after diagnosing a script error, then call resume_training.', + 'description': 'Update the training script for a run. Archives the current train.py as train_v{N}.py and ' + 'writes the new version. Use after diagnosing a script error, then call resume_training.', 'parameters': { 'type': 'object', 'properties': { - 'run_id': {'type': 'string', 'description': 'Training run ID.'}, - 'script_content': {'type': 'string', 'description': 'Full Python source code of the new training script.'}, + 'run_id': { + 'type': 'string', + 'description': 'Training run ID.' + }, + 'script_content': { + 'type': 'string', + 'description': 'Full Python source code of the new training script.' + }, }, 'required': ['run_id', 'script_content'], }, @@ -233,13 +268,16 @@ 'type': 'function', 'function': { 'name': 'list_supported_models', - 'description': 'Query the Twinkle server for its list of supported base models. Always call this before writing a training script to verify model availability.', + 'description': 'Query the Twinkle server for its list of supported base models. Always call this before ' + 'writing a training script to verify model availability.', 'parameters': { 'type': 'object', 'properties': { 'base_url': { - 'type': 'string', - 'description': 'Server base URL. Default: http://localhost:8000. Cloud: http://www.modelscope.cn/twinkle', + 'type': + 'string', + 'description': + 'Server base URL. Default: http://localhost:8000. Cloud: http://www.modelscope.cn/twinkle', }, }, 'required': [], @@ -254,8 +292,14 @@ 'parameters': { 'type': 'object', 'properties': { - 'query': {'type': 'string', 'description': 'Search query for datasets.'}, - 'limit': {'type': 'integer', 'description': 'Max results (default 5).'}, + 'query': { + 'type': 'string', + 'description': 'Search query for datasets.' + }, + 'limit': { + 'type': 'integer', + 'description': 'Max results (default 5).' + }, }, 'required': ['query'], }, @@ -269,32 +313,39 @@ 'parameters': { 'type': 'object', 'properties': { - 'query': {'type': 'string', 'description': 'Search query for models.'}, - 'limit': {'type': 'integer', 'description': 'Max results (default 5).'}, + 'query': { + 'type': 'string', + 'description': 'Search query for models.' + }, + 'limit': { + 'type': 'integer', + 'description': 'Max results (default 5).' + }, }, 'required': ['query'], }, }, }, - { 'type': 'function', 'function': { - 'name': 'get_cluster_info', - 'description': ( - 'Get cluster GPU resource info for planning training parallelism. ' - 'First attempts to query a running Ray cluster; if Ray is not available, ' - 'falls back to nvidia-smi for local GPU discovery. ' - 'The result indicates whether Ray is active — if not, the training script ' - 'should either start a local Ray cluster itself or the user should launch ' - 'Ray manually (see server mode run.sh).' - ), - 'parameters': {'type': 'object', 'properties': {}, 'required': []}, + 'name': + 'get_cluster_info', + 'description': ('Get cluster GPU resource info for planning training parallelism. ' + 'First attempts to query a running Ray cluster; if Ray is not available, ' + 'falls back to nvidia-smi for local GPU discovery. ' + 'The result indicates whether Ray is active — if not, the training script ' + 'should either start a local Ray cluster itself or the user should launch ' + 'Ray manually (see server mode run.sh).'), + 'parameters': { + 'type': 'object', + 'properties': {}, + 'required': [] + }, }, }, ] - # ────────────────────────────────────────────────────────────────────────────── # Tool executor # ────────────────────────────────────────────────────────────────────────────── @@ -325,11 +376,7 @@ async def execute(self, name: str, arguments: dict[str, Any]) -> str: def _resolve_server_url(self) -> str: """Resolve server URL: instance state > env var > default.""" - return ( - self._server_url - or os.environ.get('TWINKLE_SERVER_URL') - or 'http://localhost:8000' - ) + return (self._server_url or os.environ.get('TWINKLE_SERVER_URL') or 'http://localhost:8000') async def _tool_list_training_runs(self) -> list[dict]: return self.connection.list_training_runs() @@ -345,12 +392,12 @@ async def _tool_start_training(self, run_id: str, script_content: str, model_id: server_url = self._resolve_server_url() if not await self._check_server_health(server_url): return { - 'status': 'error', - 'run_id': run_id, - 'error': ( - f'Twinkle Server is not reachable at {server_url}. ' - 'Call start_server first to launch Ray cluster and Twinkle Server.' - ), + 'status': + 'error', + 'run_id': + run_id, + 'error': (f'Twinkle Server is not reachable at {server_url}. ' + 'Call start_server first to launch Ray cluster and Twinkle Server.'), } result = self.connection.start_training(run_id, script_content, model_id) actual_run_id = result.get('run_id', run_id) @@ -382,8 +429,8 @@ async def _tool_update_script(self, run_id: str, script_content: str) -> dict: async def _check_server_health(self, url: str) -> bool: """Check if Twinkle Server is reachable (non-blocking).""" - import urllib.request import urllib.error + import urllib.request def _probe(): try: @@ -433,8 +480,11 @@ def _start(): # Step 2: Generate server_config.yaml config_path = self._generate_server_config( - model_id=model_id, train_gpus=t_gpus, - port=port, backend=backend, samplers=sampler_list, + model_id=model_id, + train_gpus=t_gpus, + port=port, + backend=backend, + samplers=sampler_list, ) # Step 3: Start Ray cluster (multi-node GPU partitioning) @@ -449,9 +499,14 @@ def _start(): # Step 5: Wait for readiness (healthz + sampler engine) return self._wait_server_ready( - server_url=server_url, proc=proc, log_path=log_path, - sampler_list=sampler_list, model_id=model_id, - t_gpus=t_gpus, backend=backend, config_path=config_path, + server_url=server_url, + proc=proc, + log_path=log_path, + sampler_list=sampler_list, + model_id=model_id, + t_gpus=t_gpus, + backend=backend, + config_path=config_path, ) result = await asyncio.get_event_loop().run_in_executor(None, _start) @@ -468,7 +523,9 @@ def _detect_gpu_count() -> int: try: r = _sp.run( ['nvidia-smi', '--query-gpu=index', '--format=csv,noheader'], - capture_output=True, text=True, timeout=10, + capture_output=True, + text=True, + timeout=10, ) if r.returncode == 0: return len([ln for ln in r.stdout.strip().split('\n') if ln.strip()]) @@ -498,10 +555,8 @@ def _compute_gpu_allocation( needed = t_gpus + sampler_gpu_total if needed > total_hw_gpus: return { - 'error': ( - f'Requested {needed} GPUs (train={t_gpus}, samplers={sampler_gpu_total}) ' - f'but only {total_hw_gpus} available.' - ), + 'error': (f'Requested {needed} GPUs (train={t_gpus}, samplers={sampler_gpu_total}) ' + f'but only {total_hw_gpus} available.'), } return {'train_gpus': t_gpus, 'sampler_gpus': sampler_gpu_total} @@ -529,8 +584,11 @@ def _start_ray_cluster(train_gpus: int, sampler_gpus: int) -> str | None: ray_base.mkdir(parents=True, exist_ok=True) def _ray_node( - devices: str, num_gpus: int, *, - head: bool = False, node_name: str = 'worker', + devices: str, + num_gpus: int, + *, + head: bool = False, + node_name: str = 'worker', ) -> str | None: env = os.environ.copy() env['CUDA_VISIBLE_DEVICES'] = devices @@ -580,7 +638,9 @@ def _launch_server_process(config_path: str) -> tuple: cmd = ['python', '-m', 'twinkle.server', 'launch', '--config', config_path] try: proc = _sp.Popen( - cmd, stdout=log_file, stderr=_sp.STDOUT, + cmd, + stdout=log_file, + stderr=_sp.STDOUT, start_new_session=True, ) except OSError as e: @@ -601,24 +661,20 @@ def _wait_server_ready( ) -> dict: """Poll server until healthy (healthz + sampler engine ready).""" import time - import urllib.request import urllib.error + import urllib.request timeout_s = 120 if sampler_list else 60 - needed = t_gpus + sum( - s.get('gpus') or (s.get('tp', 1) * s.get('dp', 1)) for s in sampler_list - ) + needed = t_gpus + sum(s.get('gpus') or (s.get('tp', 1) * s.get('dp', 1)) for s in sampler_list) for _ in range(timeout_s): time.sleep(1) if proc.poll() is not None: # Server died — read log tail to diagnose log_tail = ToolExecutor._read_log_tail(log_path, max_chars=2000) - error_msg = ( - f'Server exited immediately (code={proc.returncode}). ' - f'Model: {model_id}, GPUs: {t_gpus}, Samplers: {len(sampler_list)}.\n' - f'--- server.log tail ---\n{log_tail}' - ) + error_msg = (f'Server exited immediately (code={proc.returncode}). ' + f'Model: {model_id}, GPUs: {t_gpus}, Samplers: {len(sampler_list)}.\n' + f'--- server.log tail ---\n{log_tail}') return { 'status': 'error', 'error': error_msg, @@ -635,31 +691,37 @@ def _wait_server_ready( return { 'status': 'started', 'warning': 'Server is up but sampler may still be loading.', - 'server_url': server_url, 'server_pid': proc.pid, - 'model_id': model_id, 'log_path': log_path, + 'server_url': server_url, + 'server_pid': proc.pid, + 'model_id': model_id, + 'log_path': log_path, } return { 'status': 'started', - 'server_url': server_url, 'server_pid': proc.pid, - 'model_id': model_id, 'train_gpus': t_gpus, + 'server_url': server_url, + 'server_pid': proc.pid, + 'model_id': model_id, + 'train_gpus': t_gpus, 'backend': backend, 'samplers': [s.get('model_id') for s in sampler_list], 'total_gpus_used': needed, - 'config_path': config_path, 'log_path': log_path, + 'config_path': config_path, + 'log_path': log_path, } return { 'status': 'timeout', 'error': 'Health check did not pass within timeout. Models may still be loading.', - 'server_pid': proc.pid, 'log_path': log_path, + 'server_pid': proc.pid, + 'log_path': log_path, } @staticmethod def _read_log_tail(log_path: str, max_chars: int = 2000) -> str: """Read the tail of a log file for error diagnosis.""" try: - with open(log_path, 'r', errors='replace') as f: + with open(log_path, errors='replace') as f: content = f.read() if len(content) <= max_chars: return content.strip() @@ -671,8 +733,8 @@ def _read_log_tail(log_path: str, max_chars: int = 2000) -> str: def _probe_sampler_ready(server_url: str, sampler_list: list[dict], fallback_model_id: str) -> bool: """Probe sampler route up to 90s to confirm vLLM engine is loaded.""" import time - import urllib.request import urllib.error + import urllib.request s_mid = sampler_list[0].get('model_id', fallback_model_id) probe_url = f'{server_url}/api/v1/sampler/{s_mid}/twinkle/create' @@ -680,7 +742,9 @@ def _probe_sampler_ready(server_url: str, sampler_list: list[dict], fallback_mod for _ in range(90): try: req = urllib.request.Request( - probe_url, method='POST', data=b'{}', + probe_url, + method='POST', + data=b'{}', headers={'Content-Type': 'application/json'}, ) urllib.request.urlopen(req, timeout=5) @@ -708,8 +772,8 @@ def _generate_server_config( - N sampler/teacher models (for RL/OPD) - 1 processor service """ - from pathlib import Path import yaml + from pathlib import Path sampler_list = samplers or [] @@ -727,11 +791,16 @@ def _short(mid: str) -> str: # 1. API Gateway applications.append({ - 'name': 'server', - 'route_prefix': '/api/v1', - 'import_path': 'server', + 'name': + 'server', + 'route_prefix': + '/api/v1', + 'import_path': + 'server', 'args': { - 'server_config': {'per_token_model_limit': 3}, + 'server_config': { + 'per_token_model_limit': 3 + }, 'supported_models': all_model_ids, }, 'deployments': [{ @@ -742,7 +811,9 @@ def _short(mid: str) -> str: 'max_replicas': 1, 'target_ongoing_requests': 128, }, - 'ray_actor_options': {'num_cpus': 0.1}, + 'ray_actor_options': { + 'num_cpus': 0.1 + }, }], }) @@ -753,48 +824,55 @@ def _short(mid: str) -> str: gpu_apps: list[tuple[int, dict]] = [] # (gpu_count, app_config) # 2a. Training model worker (student) - gpu_apps.append((train_gpus, { - 'name': f'models-{model_short}', - 'route_prefix': f'/api/v1/model/{model_id}', - 'import_path': 'model', - 'args': { - 'backend': backend, - 'model_id': f'ms://{model_id}', - 'max_length': 500000, # total tokens per forward pass (must match max_input_tokens) - 'nproc_per_node': train_gpus, - 'device_group': { - 'name': 'model', - 'ranks': train_gpus, - 'device_type': 'cuda', - }, - 'device_mesh': { - 'device_type': 'cuda', - 'dp_size': train_gpus, - }, - 'queue_config': { - 'rps_limit': 100, - 'tps_limit': 100000, - 'max_input_tokens': 500000, - }, - 'adapter_config': { - 'adapter_timeout': 600, - }, - }, - 'deployments': [{ - 'name': 'ModelManagement', - 'autoscaling_config': { - 'min_replicas': 1, - 'max_replicas': 1, - 'target_ongoing_requests': 16, - }, - 'ray_actor_options': { - 'num_cpus': 0.1, - 'runtime_env': { - 'env_vars': {'TWINKLE_TRUST_REMOTE_CODE': '1'}, + gpu_apps.append(( + train_gpus, + { + 'name': + f'models-{model_short}', + 'route_prefix': + f'/api/v1/model/{model_id}', + 'import_path': + 'model', + 'args': { + 'backend': backend, + 'model_id': f'ms://{model_id}', + 'max_length': 500000, # total tokens per forward pass (must match max_input_tokens) + 'nproc_per_node': train_gpus, + 'device_group': { + 'name': 'model', + 'ranks': train_gpus, + 'device_type': 'cuda', + }, + 'device_mesh': { + 'device_type': 'cuda', + 'dp_size': train_gpus, + }, + 'queue_config': { + 'rps_limit': 100, + 'tps_limit': 100000, + 'max_input_tokens': 500000, + }, + 'adapter_config': { + 'adapter_timeout': 600, }, }, - }], - })) + 'deployments': [{ + 'name': 'ModelManagement', + 'autoscaling_config': { + 'min_replicas': 1, + 'max_replicas': 1, + 'target_ongoing_requests': 16, + }, + 'ray_actor_options': { + 'num_cpus': 0.1, + 'runtime_env': { + 'env_vars': { + 'TWINKLE_TRUST_REMOTE_CODE': '1' + }, + }, + }, + }], + })) # 2b. Sampler/teacher models sampler_name_count: dict[str, int] = {} @@ -844,9 +922,12 @@ def _short(mid: str) -> str: mesh_config['tp_size'] = s_tp sampler_app: dict = { - 'name': s_name, - 'route_prefix': f'/api/v1/sampler/{s_model_id}', - 'import_path': 'sampler', + 'name': + s_name, + 'route_prefix': + f'/api/v1/sampler/{s_model_id}', + 'import_path': + 'sampler', 'args': { 'model_id': f'ms://{s_model_id}', 'nproc_per_node': s_total_gpus, @@ -873,7 +954,9 @@ def _short(mid: str) -> str: 'ray_actor_options': { 'num_cpus': 0.1, 'runtime_env': { - 'env_vars': {'TWINKLE_TRUST_REMOTE_CODE': '1'}, + 'env_vars': { + 'TWINKLE_TRUST_REMOTE_CODE': '1' + }, }, }, }], @@ -902,9 +985,12 @@ def _short(mid: str) -> str: # 4. Processor service applications.append({ - 'name': 'processor', - 'route_prefix': '/api/v1/processor', - 'import_path': 'processor', + 'name': + 'processor', + 'route_prefix': + '/api/v1/processor', + 'import_path': + 'processor', 'args': { 'ncpu_proc_per_node': 2, 'device_group': { @@ -924,7 +1010,9 @@ def _short(mid: str) -> str: 'max_replicas': 1, 'target_ongoing_requests': 128, }, - 'ray_actor_options': {'num_cpus': 0.1}, + 'ray_actor_options': { + 'num_cpus': 0.1 + }, }], }) @@ -990,8 +1078,8 @@ def _query(): # Use a lightweight HTTP GET instead of init_twinkle_client() which # creates a session + heartbeat thread that would leak since we never # call close(). - import urllib.request import urllib.error + import urllib.request endpoint = f'{url}/api/v1/twinkle/get_server_capabilities' req = urllib.request.Request(endpoint, method='GET') @@ -1045,10 +1133,7 @@ def _search_datasets_impl(query: str, limit: int) -> list[dict]: api = HubApi() result = api.list_datasets('', search=query, page_size=limit) datasets = result.get('datasets', []) - return [ - {'id': d.get('id', ''), 'name': d.get('display_name', d.get('id', ''))} - for d in datasets - ] + return [{'id': d.get('id', ''), 'name': d.get('display_name', d.get('id', ''))} for d in datasets] @staticmethod def _search_models_impl(query: str, limit: int) -> list[dict]: @@ -1056,7 +1141,11 @@ def _search_models_impl(query: str, limit: int) -> list[dict]: import requests resp = requests.put( 'https://modelscope.cn/api/v1/models/', - json={'Name': query, 'PageSize': limit, 'PageNumber': 1}, + json={ + 'Name': query, + 'PageSize': limit, + 'PageNumber': 1 + }, timeout=15, ) resp.raise_for_status() @@ -1064,13 +1153,10 @@ def _search_models_impl(query: str, limit: int) -> list[dict]: if not data.get('Success'): raise RuntimeError(data.get('Message', 'Unknown error')) models = data.get('Data', {}).get('Models', []) - return [ - { - 'id': f"{m.get('Path', '')}/{m.get('Name', '')}", - 'name': m.get('ChineseName') or m.get('Name', ''), - } - for m in models - ] + return [{ + 'id': f"{m.get('Path', '')}/{m.get('Name', '')}", + 'name': m.get('ChineseName') or m.get('Name', ''), + } for m in models] # ── Cluster info ── @@ -1087,11 +1173,9 @@ def _query(): # 2. Ray not available — fall back to nvidia-smi nvidia_info = self._try_nvidia_smi() nvidia_info['ray_active'] = False - nvidia_info['hint'] = ( - 'Ray cluster is not running. To use distributed training, ' - 'start Ray first: `ray start --head --num-gpus=N` or use ' - 'the server mode run.sh script.' - ) + nvidia_info['hint'] = ('Ray cluster is not running. To use distributed training, ' + 'start Ray first: `ray start --head --num-gpus=N` or use ' + 'the server mode run.sh script.') return nvidia_info return await asyncio.get_event_loop().run_in_executor(None, _query) @@ -1150,9 +1234,13 @@ def _try_nvidia_smi() -> dict: try: result = _sp.run( - ['nvidia-smi', '--query-gpu=index,name,memory.total,memory.free,utilization.gpu', - '--format=csv,noheader,nounits'], - capture_output=True, text=True, timeout=10, + [ + 'nvidia-smi', '--query-gpu=index,name,memory.total,memory.free,utilization.gpu', + '--format=csv,noheader,nounits' + ], + capture_output=True, + text=True, + timeout=10, ) if result.returncode != 0: return {'error': f'nvidia-smi failed: {result.stderr.strip()}', 'gpu_total': 0} @@ -1175,7 +1263,7 @@ def _try_nvidia_smi() -> dict: # Skip lines with unparseable values (e.g. [N/A]) continue - gpu_types = sorted(set(g['name'] for g in gpus)) + gpu_types = sorted({g['name'] for g in gpus}) return { 'gpu_total': len(gpus), 'gpu_available': len([g for g in gpus if g['utilization_pct'] < 10]), diff --git a/src/twinkle_client/auto/app.py b/src/twinkle_client/auto/app.py index cf0b8a06..48d09130 100644 --- a/src/twinkle_client/auto/app.py +++ b/src/twinkle_client/auto/app.py @@ -43,7 +43,7 @@ class TwinkleAuto: def __init__( self, - run_id: Optional[str] = None, + run_id: str | None = None, llm_base_url: str = 'http://localhost:11434/v1', llm_model: str = 'qwen3.5', llm_api_key: str = 'not-needed', @@ -64,12 +64,12 @@ def run(self) -> None: pass async def _main(self) -> None: + from openai import AsyncOpenAI + from twinkle_client.auto.agent.core import AgentLoop from twinkle_client.auto.agent.monitor import TrainingMonitor from twinkle_client.auto.connection import LocalConnection - from openai import AsyncOpenAI - # Connection self._connection = LocalConnection() if self.run_id: @@ -125,9 +125,7 @@ async def _chat_loop(self) -> None: loop = asyncio.get_event_loop() while True: try: - user_input = await loop.run_in_executor( - None, lambda: input(f'{_GREEN}You:{_RESET} ') - ) + user_input = await loop.run_in_executor(None, lambda: input(f'{_GREEN}You:{_RESET} ')) except (KeyboardInterrupt, EOFError): break diff --git a/src/twinkle_client/auto/connection.py b/src/twinkle_client/auto/connection.py index 2734cf18..cb34bc97 100644 --- a/src/twinkle_client/auto/connection.py +++ b/src/twinkle_client/auto/connection.py @@ -20,7 +20,6 @@ from __future__ import annotations import json -from twinkle.utils.logger import get_logger import os import re import shutil @@ -30,6 +29,8 @@ from pathlib import Path from typing import Any +from twinkle.utils.logger import get_logger + logger = get_logger() DEFAULT_BASE_DIR = Path.home() / '.cache' / 'twinkle' @@ -186,7 +187,11 @@ def _launch_script(self, run_id: str) -> dict[str, Any]: error_msg = output_file.read_text().strip()[-500:] if output_file.exists() else '' meta['status'] = 'error' self._write_meta(run_id, meta) - return {'status': 'error', 'run_id': run_id, 'error': error_msg or f'Process exited immediately (code={retcode})'} + return { + 'status': 'error', + 'run_id': run_id, + 'error': error_msg or f'Process exited immediately (code={retcode})' + } meta['pid'] = proc.pid meta['status'] = 'running' diff --git a/src/twinkle_client/auto/runtime.py b/src/twinkle_client/auto/runtime.py index 17cc9d5f..b3abf1ee 100644 --- a/src/twinkle_client/auto/runtime.py +++ b/src/twinkle_client/auto/runtime.py @@ -35,12 +35,11 @@ import sys import time from pathlib import Path -from typing import Any, TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from twinkle_client.model import MultiLoraTransformersModel from twinkle.dataloader import DataLoader - + from twinkle_client.model import MultiLoraTransformersModel DEFAULT_BASE_DIR = Path.home() / '.cache' / 'twinkle' @@ -68,9 +67,7 @@ def __init__(self, run_id: str | None = None, base_dir: Path | str | None = None if run_id is None: run_id = os.environ.get('TWINKLE_RUN_ID', '') if not run_id: - raise ValueError( - 'run_id must be provided or TWINKLE_RUN_ID env var must be set' - ) + raise ValueError('run_id must be provided or TWINKLE_RUN_ID env var must be set') self.run_id = run_id self.run_dir = self.base_dir / run_id @@ -247,8 +244,8 @@ def finish(self, status: str = 'completed') -> None: def register_graceful_shutdown( self, - model: 'MultiLoraTransformersModel', - dataloader: 'DataLoader | None' = None, + model: MultiLoraTransformersModel, + dataloader: DataLoader | None = None, checkpoint_name: str = 'interrupted', ) -> None: """Register SIGTERM handler for graceful shutdown with checkpoint. @@ -270,6 +267,7 @@ def register_graceful_shutdown( rt.register_graceful_shutdown(model, dataloader) # ... training loop ... """ + def _shutdown_handler(signum, frame): self.log('SIGTERM received, saving checkpoint before exit...') try: diff --git a/src/twinkle_client/common/component_rpc.py b/src/twinkle_client/common/component_rpc.py index e34c5494..78b7d7d7 100644 --- a/src/twinkle_client/common/component_rpc.py +++ b/src/twinkle_client/common/component_rpc.py @@ -30,8 +30,7 @@ def processor_base_url() -> str: def create_remote_component(processor_type: str, class_type: str, **init_kwargs: Any) -> str: """Create a server-side component and return its ``pid:``-prefixed id.""" - body = build_request( - ProcessorCreateRequest, processor_type=processor_type, class_type=class_type, **init_kwargs) + body = build_request(ProcessorCreateRequest, processor_type=processor_type, class_type=class_type, **init_kwargs) response = http_post_model(f'{processor_base_url()}/create', body) return ProcessorCreateResponse(**response.json()).processor_id diff --git a/src/twinkle_client/common/json_utils.py b/src/twinkle_client/common/json_utils.py index 51c039c1..e0517a1d 100644 --- a/src/twinkle_client/common/json_utils.py +++ b/src/twinkle_client/common/json_utils.py @@ -4,10 +4,8 @@ from collections.abc import Mapping from numbers import Number -from typing import Any - from pydantic import BaseModel - +from typing import Any _PRIMITIVE_TYPES = (str, Number, bool, bytes, type(None)) diff --git a/src/twinkle_client/data_plane.py b/src/twinkle_client/data_plane.py index 281df831..18886c42 100644 --- a/src/twinkle_client/data_plane.py +++ b/src/twinkle_client/data_plane.py @@ -10,7 +10,6 @@ from twinkle_client.http import get_base_url, http_post from twinkle_client.types.component import DataRef, DataRowsResponse - _T = TypeVar('_T') @@ -33,7 +32,11 @@ def put( ) -> DataRef: response = http_post( f'{self.server_url}/twinkle/put', - json_data={'rows': json_safe(rows), 'kind': kind, 'tags': json_safe(tags)}, + json_data={ + 'rows': json_safe(rows), + 'kind': kind, + 'tags': json_safe(tags) + }, ) response.raise_for_status() return DataRef(**response.json()) @@ -53,7 +56,10 @@ async def aput( def get(self, ref: DataRef, *, fields: list[str] | None = None) -> list[dict[str, Any]]: response = http_post( f'{self.server_url}/twinkle/get', - json_data={'ref': ref.model_dump(), 'fields': fields}, + json_data={ + 'ref': ref.model_dump(), + 'fields': fields + }, ) response.raise_for_status() return DataRowsResponse(**response.json()).rows @@ -66,7 +72,11 @@ def get_batch( ) -> DataRowsResponse: response = http_post( f'{self.server_url}/twinkle/get', - json_data={'ref': ref.model_dump(), 'fields': fields, 'include_tags': True}, + json_data={ + 'ref': ref.model_dump(), + 'fields': fields, + 'include_tags': True + }, ) response.raise_for_status() return DataRowsResponse(**response.json()) diff --git a/src/twinkle_client/dataloader/dataloader.py b/src/twinkle_client/dataloader/dataloader.py index f9798501..7b0b7cf5 100644 --- a/src/twinkle_client/dataloader/dataloader.py +++ b/src/twinkle_client/dataloader/dataloader.py @@ -6,7 +6,7 @@ from twinkle_client.common.component_rpc import call_remote_component, create_remote_component -class DataLoader(object): +class DataLoader: """Client wrapper for DataLoader that calls server HTTP endpoints.""" def __init__(self, dataset: Union[Dataset, Callable], **kwargs): diff --git a/src/twinkle_client/dataset/base.py b/src/twinkle_client/dataset/base.py index f52bf3ab..160e521e 100644 --- a/src/twinkle_client/dataset/base.py +++ b/src/twinkle_client/dataset/base.py @@ -7,7 +7,7 @@ from twinkle_client.common.component_rpc import call_remote_component, create_remote_component -class Dataset(object): +class Dataset: """Client wrapper for Dataset that calls server HTTP endpoints.""" def __init__(self, dataset_meta: DatasetMeta = None, **kwargs): diff --git a/src/twinkle_client/exceptions.py b/src/twinkle_client/exceptions.py index de91bbd9..2e61c476 100644 --- a/src/twinkle_client/exceptions.py +++ b/src/twinkle_client/exceptions.py @@ -15,9 +15,8 @@ """ from __future__ import annotations -from typing import Any, Optional - import requests +from typing import Any, Optional class TwinkleClientValidationError(ValueError): @@ -44,10 +43,10 @@ class TwinkleHTTPError(requests.HTTPError): def __init__( self, *args: Any, - status_code: Optional[int] = None, - error_code: Optional[int] = None, + status_code: int | None = None, + error_code: int | None = None, category: str = 'Unknown', - request_id: Optional[str] = None, + request_id: str | None = None, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) @@ -72,8 +71,8 @@ def __init__( *, category: str, request_id: str, - error_code: Optional[int] = None, - details: Optional[list[dict[str, Any]]] = None, + error_code: int | None = None, + details: list[dict[str, Any]] | None = None, ) -> None: super().__init__(error) self.error = error @@ -96,7 +95,7 @@ def __init__( error: str, *, request_id: str, - error_code: Optional[int] = None, + error_code: int | None = None, ) -> None: super().__init__(error) self.error = error diff --git a/src/twinkle_client/http/__init__.py b/src/twinkle_client/http/__init__.py index 068cfa51..9ea02f3b 100644 --- a/src/twinkle_client/http/__init__.py +++ b/src/twinkle_client/http/__init__.py @@ -1,6 +1,6 @@ from .http_utils import http_delete, http_get, http_post, http_post_model -from .utils import (TWINKLE_SERVER_TOKEN, TWINKLE_SERVER_URL, get_api_key, get_base_url, get_request_id, - get_session_id, set_api_key, set_base_url, set_request_id, set_session_id) +from .utils import (TWINKLE_SERVER_TOKEN, TWINKLE_SERVER_URL, get_api_key, get_base_url, get_request_id, get_session_id, + set_api_key, set_base_url, set_request_id, set_session_id) __all__ = [ 'http_get', diff --git a/src/twinkle_client/http/http_utils.py b/src/twinkle_client/http/http_utils.py index a96a52fd..766739e9 100644 --- a/src/twinkle_client/http/http_utils.py +++ b/src/twinkle_client/http/http_utils.py @@ -83,10 +83,8 @@ def _handle_response(response: requests.Response) -> requests.Response: summary = body.get('error') or body.get('detail') or response.text else: category, error_code, request_id, summary = 'Unknown', None, None, response.text - http_error_msg = ( - f'{response.status_code} Error for url: {response.url}\n' - f'Server detail:\n{summary}' - ) + http_error_msg = (f'{response.status_code} Error for url: {response.url}\n' + f'Server detail:\n{summary}') raise TwinkleHTTPError( http_error_msg, response=response, diff --git a/src/twinkle_client/manager.py b/src/twinkle_client/manager.py index 12257d63..f08e2a16 100644 --- a/src/twinkle_client/manager.py +++ b/src/twinkle_client/manager.py @@ -4,16 +4,18 @@ import atexit import threading from typing import Any, Dict, List, Optional, Tuple + from twinkle import get_logger -from twinkle_client.types.server import (CapacityInfoResponse, DeleteCheckpointResponse, GetServerCapabilitiesResponse) +from twinkle_client.types.server import CapacityInfoResponse, DeleteCheckpointResponse, GetServerCapabilitiesResponse from twinkle_client.types.session import (CreateSessionRequest, CreateSessionResponse, SessionHeartbeatRequest, - SessionHeartbeatResponse) + SessionHeartbeatResponse) from twinkle_client.types.training import (Checkpoint, Cursor, ParsedCheckpointTwinklePath, TrainingRun, - TrainingRunsResponse, WeightsInfoResponse) + TrainingRunsResponse, WeightsInfoResponse) from .http import get_api_key, get_base_url, http_delete, http_get, http_post, set_api_key, set_base_url, set_session_id logger = get_logger() + class TwinkleClientError(Exception): """Base exception for TwinkleManager errors.""" pass @@ -44,11 +46,11 @@ class TwinkleClient: def __init__( self, - base_url: Optional[str] = None, - api_key: Optional[str] = None, - route_prefix: Optional[str] = '/twinkle', + base_url: str | None = None, + api_key: str | None = None, + route_prefix: str | None = '/twinkle', session_heartbeat_interval: int = 10, - session_metadata: Optional[Dict[str, Any]] = None, + session_metadata: dict[str, Any] | None = None, ): # Resolve and store config, then propagate to context so all generated # client objects that call get_base_url() / get_api_key() get these values. @@ -110,7 +112,7 @@ def _handle_response(self, response, expected_code: int = 200) -> dict[str, Any] raise TwinkleClientError(f'Request failed with status {response.status_code}: {detail}') return response.json() - def create_session(self, metadata: Optional[Dict[str, Any]] = None) -> str: + def create_session(self, metadata: dict[str, Any] | None = None) -> str: """ Create a server-side session. @@ -201,7 +203,7 @@ def get_server_capabilities(self) -> GetServerCapabilitiesResponse: # Training Runs # ------------------------------------------------------------------ - def list_training_runs(self, limit: int = 20, offset: int = 0, all_users: bool = False) -> List[TrainingRun]: + def list_training_runs(self, limit: int = 20, offset: int = 0, all_users: bool = False) -> list[TrainingRun]: """ List training runs. @@ -218,7 +220,7 @@ def list_training_runs(self, limit: int = 20, offset: int = 0, all_users: bool = Raises: TwinkleClientError: If the request fails. """ - params: Dict[str, Any] = {'limit': limit, 'offset': offset} + params: dict[str, Any] = {'limit': limit, 'offset': offset} if all_users: params['all_users'] = 'true' @@ -232,7 +234,7 @@ def list_training_runs_with_cursor( limit: int = 20, offset: int = 0, all_users: bool = False, - ) -> Tuple[List[TrainingRun], Cursor]: + ) -> tuple[list[TrainingRun], Cursor]: """ List training runs with pagination info. @@ -247,7 +249,7 @@ def list_training_runs_with_cursor( Raises: TwinkleClientError: If the request fails. """ - params: Dict[str, Any] = {'limit': limit, 'offset': offset} + params: dict[str, Any] = {'limit': limit, 'offset': offset} if all_users: params['all_users'] = 'true' @@ -279,7 +281,7 @@ def get_training_run(self, run_id: str) -> TrainingRun: # Checkpoints # ------------------------------------------------------------------ - def list_checkpoints(self, run_id: str) -> List[Checkpoint]: + def list_checkpoints(self, run_id: str) -> list[Checkpoint]: """ List checkpoints for a training run. @@ -382,7 +384,7 @@ def get_weights_info(self, twinkle_path: str) -> WeightsInfoResponse: # Convenience Methods # ------------------------------------------------------------------ - def get_latest_checkpoint_path(self, run_id: str) -> Optional[str]: + def get_latest_checkpoint_path(self, run_id: str) -> str | None: """ Get the filesystem path to the latest checkpoint for a training run. @@ -403,7 +405,7 @@ def get_latest_checkpoint_path(self, run_id: str) -> Optional[str]: latest = checkpoints[-1] return self.get_checkpoint_path(run_id, latest.checkpoint_id).path - def find_training_run_by_model(self, base_model: str) -> List[TrainingRun]: + def find_training_run_by_model(self, base_model: str) -> list[TrainingRun]: """ Find training runs for a specific base model. diff --git a/src/twinkle_client/model/multi_lora_transformers.py b/src/twinkle_client/model/multi_lora_transformers.py index a32029a9..b2dbdf88 100644 --- a/src/twinkle_client/model/multi_lora_transformers.py +++ b/src/twinkle_client/model/multi_lora_transformers.py @@ -51,9 +51,7 @@ def __init__(self, model_id: str, **kwargs): # applied at most once. Reserved once per call and reused on retry. self._seq_counter = itertools.count(1) self._seq_lock = threading.Lock() - response = http_post( - url=f'{self.server_url}/create', - ) + response = http_post(url=f'{self.server_url}/create', ) response.raise_for_status() # ------------------------------------------------------------------ # @@ -252,12 +250,7 @@ def forward_backward_from_data_plane( def step(self, **kwargs) -> None: """Execute optimizer step.""" self._submit( - 'step', - model_types.StepRequest, - None, - adapter_name=self.adapter_name, - seq_id=self._next_seq_id(), - **kwargs) + 'step', model_types.StepRequest, None, adapter_name=self.adapter_name, seq_id=self._next_seq_id(), **kwargs) def zero_grad(self, **kwargs) -> None: """Zero out gradients.""" diff --git a/src/twinkle_client/processor/base.py b/src/twinkle_client/processor/base.py index 45502209..5d8d9852 100644 --- a/src/twinkle_client/processor/base.py +++ b/src/twinkle_client/processor/base.py @@ -6,7 +6,7 @@ from twinkle_client.common.component_rpc import call_remote_component, create_remote_component -class InputProcessor(object): +class InputProcessor: """Client wrapper for InputProcessor that calls server HTTP endpoints.""" def __init__(self, diff --git a/src/twinkle_client/rollout/multi_turn.py b/src/twinkle_client/rollout/multi_turn.py index c20597af..54365cb7 100644 --- a/src/twinkle_client/rollout/multi_turn.py +++ b/src/twinkle_client/rollout/multi_turn.py @@ -104,10 +104,8 @@ def __call__(self, trajectories: List[Trajectory], **kwargs) -> List[Trajectory] if n == 0: return [] - sampling_params = self._as_sampling_params_dict( - kwargs.get('sampling_params', self.sampling_params)) - tool_managers = self._resolve_tool_managers( - kwargs.get('tool_manager', self.tool_manager), n) + sampling_params = self._as_sampling_params_dict(kwargs.get('sampling_params', self.sampling_params)) + tool_managers = self._resolve_tool_managers(kwargs.get('tool_manager', self.tool_manager), n) # 1. Encode each trajectory once; ``pifs[i]`` is the live per-turn # state for trajectory ``i``. ``vLLMSampler.sample`` is responsible for @@ -153,10 +151,9 @@ def __call__(self, trajectories: List[Trajectory], **kwargs) -> List[Trajectory] # carries ``input_ids``. A missing feature makes the next round # impossible, so raise a batch/trajectory-indexed RuntimeError. if seq.new_input_feature is None or 'input_ids' not in seq.new_input_feature: - raise RuntimeError( - f'Sampler returned a sequence without new_input_feature.input_ids at ' - f'batch index {local_idx} (trajectory {global_idx}); ' - f'cannot continue multi-turn.') + raise RuntimeError(f'Sampler returned a sequence without new_input_feature.input_ids at ' + f'batch index {local_idx} (trajectory {global_idx}); ' + f'cannot continue multi-turn.') pifs[global_idx] = dict(seq.new_input_feature) # Per-round logprobs/token alignment guard: each sampled token @@ -165,10 +162,9 @@ def __call__(self, trajectories: List[Trajectory], **kwargs) -> List[Trajectory] # Ray paths cannot drift on this invariant. if seq.logprobs is not None: if len(seq.logprobs) != len(seq.tokens): - raise RuntimeError( - f'logprobs length ({len(seq.logprobs)}) does not match sampled ' - f'token count ({len(seq.tokens)}) at turn {turns[global_idx]} ' - f'(trajectory {global_idx})') + raise RuntimeError(f'logprobs length ({len(seq.logprobs)}) does not match sampled ' + f'token count ({len(seq.tokens)}) at turn {turns[global_idx]} ' + f'(trajectory {global_idx})') all_logprobs[global_idx].extend(seq.logprobs) stop_reasons[global_idx] = seq.stop_reason @@ -183,8 +179,8 @@ def __call__(self, trajectories: List[Trajectory], **kwargs) -> List[Trajectory] continue # 3a. Sequence-length cap. - if (self.max_trajectory_tokens is not None and len( - pifs[global_idx].get('input_ids') or []) >= self.max_trajectory_tokens): + if (self.max_trajectory_tokens is not None + and len(pifs[global_idx].get('input_ids') or []) >= self.max_trajectory_tokens): truncated[global_idx] = True done[global_idx] = True continue @@ -211,9 +207,8 @@ def __call__(self, trajectories: List[Trajectory], **kwargs) -> List[Trajectory] # 4. Dispatch tools for this trajectory via its ToolManager. tool_manager = tool_managers[global_idx] if tool_manager is None: - raise ValueError( - f'trajectory {global_idx} produced tool_calls but no tool_manager ' - f'was provided (at construction time or as a per-call kwarg).') + raise ValueError(f'trajectory {global_idx} produced tool_calls but no tool_manager ' + f'was provided (at construction time or as a per-call kwarg).') tool_messages = [{ 'role': 'tool', 'content': tool_manager(tc), diff --git a/src/twinkle_client/sampler/vllm_sampler.py b/src/twinkle_client/sampler/vllm_sampler.py index 604032bb..648c4b34 100644 --- a/src/twinkle_client/sampler/vllm_sampler.py +++ b/src/twinkle_client/sampler/vllm_sampler.py @@ -1,8 +1,7 @@ import asyncio from dataclasses import asdict -from typing import Any, Dict, List, Optional, Union - from peft import PeftConfig +from typing import Any, Dict, List, Optional, Union from twinkle.data_format import InputFeature, SamplingParams, Trajectory from twinkle_client._request_builder import build_request @@ -48,10 +47,7 @@ def __init__(self, model_id: str, **kwargs): model_id = model_id.split('://')[1] self.model_id = model_id self.server_url = f'{self.server_url}/sampler/{model_id}/twinkle' - response = http_post( - url=f'{self.server_url}/create', - json_data=kwargs - ) + response = http_post(url=f'{self.server_url}/create', json_data=kwargs) response.raise_for_status() @staticmethod @@ -128,11 +124,7 @@ def sample_to_data_plane( num_samples: int = 1, ) -> DataRef: """Generate complete prompt groups and keep their rows in the server DataPlane.""" - source = ({ - 'input_ref': inputs.model_dump() - } if isinstance(inputs, DataRef) else { - 'inputs': _json_safe(inputs) - }) + source = ({'input_ref': inputs.model_dump()} if isinstance(inputs, DataRef) else {'inputs': _json_safe(inputs)}) body = build_request( DataPlaneSampleRequest, sampling_params=_json_safe(sampling_params) if sampling_params else None, @@ -188,9 +180,8 @@ async def asample_to_data_plane( def unload_adapter_paths(self, adapter_paths: list[str]) -> None: """Evict policy snapshots that are no longer referenced by this client.""" - http_post_model( - f'{self.server_url}/unload_adapter_paths', - build_request(UnloadAdapterPathsRequest, adapter_paths=adapter_paths)) + http_post_model(f'{self.server_url}/unload_adapter_paths', + build_request(UnloadAdapterPathsRequest, adapter_paths=adapter_paths)) def set_template(self, template_cls: str, adapter_name: str = '', **kwargs) -> SamplerSetTemplateResponse: """Set the template for encoding trajectories.""" @@ -201,6 +192,5 @@ def set_template(self, template_cls: str, adapter_name: str = '', **kwargs) -> S def apply_patch(self, patch_cls: str, **kwargs) -> None: """Apply a patch to the model.""" from twinkle_client.types.model import ApplyPatchRequest - body = build_request( - ApplyPatchRequest, patch_cls=patch_cls, adapter_name=self.adapter_name or '', **kwargs) + body = build_request(ApplyPatchRequest, patch_cls=patch_cls, adapter_name=self.adapter_name or '', **kwargs) http_post_model(f'{self.server_url}/apply_patch', body) diff --git a/src/twinkle_client/skills/base.py b/src/twinkle_client/skills/base.py index cf2bad78..1a64c7d2 100644 --- a/src/twinkle_client/skills/base.py +++ b/src/twinkle_client/skills/base.py @@ -14,10 +14,11 @@ from __future__ import annotations import dataclasses -from twinkle.utils.logger import get_logger from abc import ABC, abstractmethod from pathlib import Path +from twinkle.utils.logger import get_logger + logger = get_logger() # File stems to skip when scanning for skill markdown files diff --git a/src/twinkle_client/skills/manager.py b/src/twinkle_client/skills/manager.py index 34ceb9c5..6de4484d 100644 --- a/src/twinkle_client/skills/manager.py +++ b/src/twinkle_client/skills/manager.py @@ -4,7 +4,6 @@ from __future__ import annotations from twinkle.utils.logger import get_logger - from twinkle_client.skills.base import Skill, SkillProvider logger = get_logger() @@ -72,10 +71,8 @@ def format_for_prompt(self) -> str: sections: list[str] = [] sections.append('# Available Skills') sections.append('') - sections.append( - 'The following skills provide you with specialized knowledge and capabilities. ' - 'Use them to better assist the user.' - ) + sections.append('The following skills provide you with specialized knowledge and capabilities. ' + 'Use them to better assist the user.') sections.append('') for skill in self._skills: diff --git a/src/twinkle_client/skills/modelscope_provider.py b/src/twinkle_client/skills/modelscope_provider.py index 9fe27f9c..3e83c001 100644 --- a/src/twinkle_client/skills/modelscope_provider.py +++ b/src/twinkle_client/skills/modelscope_provider.py @@ -4,9 +4,9 @@ from __future__ import annotations import asyncio -from twinkle.utils.logger import get_logger from pathlib import Path +from twinkle.utils.logger import get_logger from twinkle_client.skills.base import SkillProvider logger = get_logger() @@ -42,7 +42,11 @@ async def fetch(self) -> None: if (repo_dir / '.git').exists(): proc = await asyncio.create_subprocess_exec( - 'git', '-C', str(repo_dir), 'pull', '--ff-only', + 'git', + '-C', + str(repo_dir), + 'pull', + '--ff-only', stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) @@ -52,8 +56,14 @@ async def fetch(self) -> None: else: self.cache_dir.mkdir(parents=True, exist_ok=True) proc = await asyncio.create_subprocess_exec( - 'git', 'clone', '--depth', '1', '--branch', self._branch, - self._repo_url, str(repo_dir), + 'git', + 'clone', + '--depth', + '1', + '--branch', + self._branch, + self._repo_url, + str(repo_dir), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) diff --git a/src/twinkle_client/types/__init__.py b/src/twinkle_client/types/__init__.py index 9270658a..d272f9ff 100644 --- a/src/twinkle_client/types/__init__.py +++ b/src/twinkle_client/types/__init__.py @@ -1,129 +1,33 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -from .model import ( - AddAdapterRequest, - AddMetricRequest, - AddMetricResponse, - AdapterRequest, - ApplyPatchRequest, - ApplyPatchResponse, - BackwardResponse, - CalculateLossResponse, - CalculateMetricRequest, - CalculateMetricResponse, - ClipGradAndStepRequest, - ClipGradAndStepResponse, - ClipGradNormRequest, - ClipGradNormResponse, - CreateRequest, - CreateResponse, - DataPlaneForwardOnlyRequest, - DataPlaneForwardRequest, - ForwardBackwardResponse, - ForwardBackwardTaskRequest, - ForwardOnlyRequest, - ForwardRequest, - ForwardResponse, - GetTrainConfigsResponse, - LoadRequest, - LoadResponse, - LrStepRequest, - LrStepResponse, - ModelResult, - OkResponse, - ResumeFromCheckpointRequest, - SaveRequest, - SaveResponse, - SetLossRequest, - SetLossResponse, - SetLrSchedulerRequest, - SetLrSchedulerResponse, - SetOptimizerRequest, - SetOptimizerResponse, - SetProcessorRequest, - SetProcessorResponse, - SetTemplateRequest, - SetTemplateResponse, - StepRequest, - StepResponse, - TrainingProgressResponse, - UploadToHubRequest, - ZeroGradResponse, -) -from .processor import ( - ProcessorCallRequest, - ProcessorCallResponse, - ProcessorCreateRequest, - ProcessorCreateResponse, - ProcessorHeartbeatRequest, - ProcessorHeartbeatResponse, -) -from .sampler import ( - SampledSequenceModel, - SampleRequest, - SampleResponseModel, - SampleResponseModelList, - SamplerAddAdapterRequest, - SamplerAddAdapterResponse, - SamplerCreateResponse, - SamplerSetTemplateRequest, - SamplerSetTemplateResponse, -) -from .server import ( - CheckpointPathResponse, - DeleteCheckpointResponse, - ErrorResponse, - GetServerCapabilitiesResponse, - HealthResponse, - SupportedModel, - WeightsInfoRequest, - CapacityInfoResponse, -) +# yapf: disable +from .base import (BACKEND_ONLY_KEY, DataModel, FieldRole, ResponseModel, StrictRequest, backend_kwarg, backend_only, + fields_with_role, passthrough, read_backend_only, read_field_role) +from .checkpoint import ResolvedLoadPath +from .component import (DataAppendRequest, DataGetRequest, DataPlaneSampleRequest, DataPutRequest, DataRef, + DataReleaseRequest, DataRowsResponse, UnloadAdapterPathsRequest) +from .data import (CORE_INPUT_KEYS, VLM_TENSOR_FIELDS, WireInputBatch, WireInputFeature, WireInputs, WireMessage, + WireTrajectory, declared_wire_keys, export_batch) +from .lifecycle import TERMINAL_STATUSES, CancelRequest, CancelResponse, RetrieveFutureRequest, TaskEnvelope, TaskStatus +from .model import (AdapterRequest, AddAdapterRequest, AddMetricRequest, AddMetricResponse, ApplyPatchRequest, + ApplyPatchResponse, BackwardResponse, CalculateLossResponse, CalculateMetricRequest, + CalculateMetricResponse, ClipGradAndStepRequest, ClipGradAndStepResponse, ClipGradNormRequest, + ClipGradNormResponse, CreateRequest, CreateResponse, DataPlaneForwardOnlyRequest, + DataPlaneForwardRequest, ForwardBackwardResponse, ForwardBackwardTaskRequest, ForwardOnlyRequest, + ForwardRequest, ForwardResponse, GetTrainConfigsResponse, LoadRequest, LoadResponse, LrStepRequest, + LrStepResponse, ModelResult, OkResponse, ResumeFromCheckpointRequest, SaveRequest, SaveResponse, + SetLossRequest, SetLossResponse, SetLrSchedulerRequest, SetLrSchedulerResponse, SetOptimizerRequest, + SetOptimizerResponse, SetProcessorRequest, SetProcessorResponse, SetTemplateRequest, + SetTemplateResponse, StepRequest, StepResponse, TrainingProgressResponse, UploadToHubRequest, + ZeroGradResponse) +from .processor import (ProcessorCallRequest, ProcessorCallResponse, ProcessorCreateRequest, ProcessorCreateResponse, + ProcessorHeartbeatRequest, ProcessorHeartbeatResponse) +from .sampler import (SampledSequenceModel, SamplerAddAdapterRequest, SamplerAddAdapterResponse, SamplerCreateResponse, + SampleRequest, SampleResponseModel, SampleResponseModelList, SamplerSetTemplateRequest, + SamplerSetTemplateResponse) +from .server import (CapacityInfoResponse, CheckpointPathResponse, DeleteCheckpointResponse, ErrorResponse, + GetServerCapabilitiesResponse, HealthResponse, SupportedModel, WeightsInfoRequest) from .session import CreateSessionRequest, CreateSessionResponse, SessionHeartbeatRequest, SessionHeartbeatResponse -from .training import ( - Checkpoint, - CheckpointsListResponse, - CreateModelRequest, - Cursor, - LoraConfig, - ParsedCheckpointTwinklePath, - TrainingRun, - TrainingRunsResponse, - WeightsInfoResponse, -) +from .training import (Checkpoint, CheckpointsListResponse, CreateModelRequest, Cursor, LoraConfig, + ParsedCheckpointTwinklePath, TrainingRun, TrainingRunsResponse, WeightsInfoResponse) -from .checkpoint import ResolvedLoadPath -from .base import ( - BACKEND_ONLY_KEY, - DataModel, - FieldRole, - ResponseModel, - StrictRequest, - backend_kwarg, - backend_only, - fields_with_role, - passthrough, - read_backend_only, - read_field_role, -) -from .data import ( - CORE_INPUT_KEYS, - VLM_TENSOR_FIELDS, - WireInputBatch, - WireInputFeature, - WireInputs, - WireMessage, - WireTrajectory, - declared_wire_keys, - export_batch, -) -from .lifecycle import CancelRequest, CancelResponse, RetrieveFutureRequest, TaskEnvelope, TaskStatus, TERMINAL_STATUSES -from .component import ( - DataAppendRequest, - DataGetRequest, - DataPlaneSampleRequest, - DataPutRequest, - DataRef, - DataReleaseRequest, - DataRowsResponse, - UnloadAdapterPathsRequest, -) +# yapf: enable diff --git a/src/twinkle_client/types/base.py b/src/twinkle_client/types/base.py index 4737a098..19330327 100644 --- a/src/twinkle_client/types/base.py +++ b/src/twinkle_client/types/base.py @@ -158,7 +158,7 @@ def read_field_role(field_info: FieldInfo) -> FieldRole: return FieldRole(value) if value is not None else FieldRole.Control -def read_backend_only(field_info: FieldInfo) -> Optional[tuple[str, ...]]: +def read_backend_only(field_info: FieldInfo) -> tuple[str, ...] | None: """Return the backend tuple a field was restricted to, or ``None`` if unrestricted.""" value = _read_extra(field_info, BACKEND_ONLY_KEY) return tuple(value) if value else None diff --git a/src/twinkle_client/types/component.py b/src/twinkle_client/types/component.py index 4c7762ee..4751774e 100644 --- a/src/twinkle_client/types/component.py +++ b/src/twinkle_client/types/component.py @@ -28,19 +28,19 @@ class DataRef(BaseModel): class DataPutRequest(StrictRequest): rows: list[dict[str, Any]] kind: str = 'data' - tags: Optional[list[dict[str, Any]]] = None + tags: list[dict[str, Any]] | None = None class DataGetRequest(StrictRequest): ref: DataRef - fields: Optional[list[str]] = None + fields: list[str] | None = None include_tags: bool = False class DataAppendRequest(StrictRequest): ref: DataRef rows: list[dict[str, Any]] - tags: Optional[list[dict[str, Any]]] = None + tags: list[dict[str, Any]] | None = None class DataReleaseRequest(StrictRequest): @@ -58,17 +58,17 @@ class DataPlaneSampleRequest(StrictRequest): Exactly one input source: inline entries (wire-validated) or a ``DataRef``. """ - inputs: Optional[WireInputBatch] = None - input_ref: Optional[DataRef] = None - sampling_params: Optional[dict[str, JsonValue]] = None + inputs: WireInputBatch | None = None + input_ref: DataRef | None = None + sampling_params: dict[str, JsonValue] | None = None adapter_name: str = '' - adapter_uri: Optional[str] = None - policy_version: Optional[int] = None - group_ids: Optional[list[str]] = None + adapter_uri: str | None = None + policy_version: int | None = None + group_ids: list[str] | None = None num_samples: int = Field(default=1, ge=1) @model_validator(mode='after') - def validate_input(self) -> 'DataPlaneSampleRequest': + def validate_input(self) -> DataPlaneSampleRequest: if (self.inputs is None) == (self.input_ref is None): raise ValueError('exactly one of inputs and input_ref must be provided') if self.group_ids is not None and self.inputs is not None: diff --git a/src/twinkle_client/types/data.py b/src/twinkle_client/types/data.py index f7bda96b..fb8c6e98 100644 --- a/src/twinkle_client/types/data.py +++ b/src/twinkle_client/types/data.py @@ -42,8 +42,8 @@ from pydantic import BeforeValidator, Field, StrictInt, model_validator from typing import Annotated, Any, List, Literal, Optional, Union -from twinkle_client.types.base import DataModel from twinkle.data_format.encoding import ENCODED_INPUT_KEYS +from twinkle_client.types.base import DataModel # --------------------------------------------------------------------------- # # Leaf types. Shallowest-first, and ``StrictInt`` wherever the values come from a @@ -89,43 +89,43 @@ class WireMessage(DataModel): """One conversation turn, as sent over HTTP.""" - role: Optional[Literal['system', 'user', 'assistant', 'tool']] = None - type: Optional[str] = None - content: Optional[Union[str, List[dict[str, Any]]]] = None - tool_calls: Optional[List[dict[str, Any]]] = None - tool_call_id: Optional[str] = None - reasoning_content: Optional[str] = None - images: Optional[MediaList] = None - videos: Optional[MediaList] = None - audios: Optional[MediaList] = None + role: Literal['system', 'user', 'assistant', 'tool'] | None = None + type: str | None = None + content: str | list[dict[str, Any]] | None = None + tool_calls: list[dict[str, Any]] | None = None + tool_call_id: str | None = None + reasoning_content: str | None = None + images: MediaList | None = None + videos: MediaList | None = None + audios: MediaList | None = None class WireInputFeature(DataModel): """An already-encoded entry: token ids (or embeddings) plus aligned tensors.""" - input_ids: Optional[Ints1to2] = None - input_embedding: Optional[Numbers1to2] = None - attention_mask: Optional[Ints1to2] = None - labels: Optional[Ints1to2] = None - completion_mask: Optional[Ints1to2] = None + input_ids: Ints1to2 | None = None + input_embedding: Numbers1to2 | None = None + attention_mask: Ints1to2 | None = None + labels: Ints1to2 | None = None + completion_mask: Ints1to2 | None = None # 1-D standard encoding, 2-D Qwen-VL mrope ``[3, T]``, 3-D megatron ``[3, 1, N]``. - position_ids: Optional[Ints1to3] = None + position_ids: Ints1to3 | None = None # Exactly ``[seq_len, num_layers, topk]``. - routed_experts: Optional[Ints3] = None - length: Optional[StrictInt] = None + routed_experts: Ints3 | None = None + length: StrictInt | None = None # VLM tensors: float values are normal here, so no strict-int leaves. - pixel_values: Optional[Numbers1to4] = None - image_grid_thw: Optional[Numbers1to4] = None - pixel_values_videos: Optional[Numbers1to4] = None - video_grid_thw: Optional[Numbers1to4] = None - input_features: Optional[Numbers1to4] = None - input_features_mask: Optional[Numbers1to4] = None - feature_attention_mask: Optional[Numbers1to4] = None - grid_thws: Optional[Numbers1to4] = None + pixel_values: Numbers1to4 | None = None + image_grid_thw: Numbers1to4 | None = None + pixel_values_videos: Numbers1to4 | None = None + video_grid_thw: Numbers1to4 | None = None + input_features: Numbers1to4 | None = None + input_features_mask: Numbers1to4 | None = None + feature_attention_mask: Numbers1to4 | None = None + grid_thws: Numbers1to4 | None = None @model_validator(mode='after') - def require_encoded_key(self) -> 'WireInputFeature': + def require_encoded_key(self) -> WireInputFeature: """At least one of the encoded-input keys must be present. Declared as a model validator rather than by making ``input_ids`` required: @@ -140,14 +140,14 @@ def require_encoded_key(self) -> 'WireInputFeature': class WireTrajectory(DataModel): """A not-yet-encoded entry: messages the server template will encode.""" - messages: List[WireMessage] - images: Optional[MediaList] = None - videos: Optional[MediaList] = None - audios: Optional[MediaList] = None - tools: Optional[List[dict[str, Any]]] = None + messages: list[WireMessage] + images: MediaList | None = None + videos: MediaList | None = None + audios: MediaList | None = None + tools: list[dict[str, Any]] | None = None # ``List[Tuple[str, str]]`` on the wire: the PyArrow-stable encoding of the # user-data pairs attached by ``twinkle.data_format.attach_user_data``. - user_data: Optional[List[tuple[str, str]]] = None + user_data: list[tuple[str, str]] | None = None # A batch is homogeneous: every entry is encoded, or none is. Expressed as a union of @@ -199,7 +199,7 @@ def declared_wire_keys() -> frozenset[str]: return frozenset(WireInputFeature.model_fields) | frozenset(WireTrajectory.model_fields) -def export(entry: Union[WireInputFeature, WireTrajectory]) -> dict[str, Any]: +def export(entry: WireInputFeature | WireTrajectory) -> dict[str, Any]: """Render a validated entry as the plain dict the backend consumes. ``exclude_none=True`` is required, not cosmetic: Twinkle_Core branches on key @@ -211,6 +211,6 @@ def export(entry: Union[WireInputFeature, WireTrajectory]) -> dict[str, Any]: return entry.model_dump(exclude_none=True) -def export_batch(entries: List[Any]) -> List[dict[str, Any]]: +def export_batch(entries: list[Any]) -> list[dict[str, Any]]: """Export a validated batch, leaving already-plain entries untouched.""" return [export(entry) if isinstance(entry, DataModel) else entry for entry in entries] diff --git a/src/twinkle_client/types/errors.py b/src/twinkle_client/types/errors.py index f1f4e3e1..b896e4ee 100644 --- a/src/twinkle_client/types/errors.py +++ b/src/twinkle_client/types/errors.py @@ -51,8 +51,8 @@ class ErrorPayload(ResponseModel): category: ErrorCategory error_code: int = Field(ge=400, le=599) request_id: str - traceback: Optional[str] = Field(default=None, max_length=65536) - details: Optional[list[dict[str, Any]]] = None + traceback: str | None = Field(default=None, max_length=65536) + details: list[dict[str, Any]] | None = None @field_validator('category', mode='before') @classmethod @@ -62,7 +62,7 @@ def normalize_legacy_category(cls, value: Any) -> Any: return value @model_validator(mode='after') - def traceback_is_server_only(self) -> 'ErrorPayload': + def traceback_is_server_only(self) -> ErrorPayload: if self.traceback is not None and self.category is not ErrorCategory.Server: raise ValueError('traceback is only valid for server errors') return self diff --git a/src/twinkle_client/types/lifecycle.py b/src/twinkle_client/types/lifecycle.py index ecff212a..911f7a0e 100644 --- a/src/twinkle_client/types/lifecycle.py +++ b/src/twinkle_client/types/lifecycle.py @@ -77,7 +77,7 @@ class TaskEnvelope(ResponseModel): request_id: str status: TaskStatus - result: Optional[Any] = None # set iff status == 'completed' - error: Optional[ErrorPayload] = None # set iff status == 'failed' - queue_state: Optional[QueueStateLiteral] = None - queue_state_reason: Optional[str] = None + result: Any | None = None # set iff status == 'completed' + error: ErrorPayload | None = None # set iff status == 'failed' + queue_state: QueueStateLiteral | None = None + queue_state_reason: str | None = None diff --git a/src/twinkle_client/types/model.py b/src/twinkle_client/types/model.py index 554b4f1e..40c1ffcd 100644 --- a/src/twinkle_client/types/model.py +++ b/src/twinkle_client/types/model.py @@ -47,14 +47,14 @@ class AdapterRequest(StrictRequest): """ adapter_name: str - seq_id: Optional[int] = None - gradient_accumulation_steps: Optional[int] = backend_kwarg(default=None, ge=1) + seq_id: int | None = None + gradient_accumulation_steps: int | None = backend_kwarg(default=None, ge=1) class StepRequest(AdapterRequest): """Body of ``POST /twinkle/step``.""" - optim_params: Optional[Dict[str, JsonValue]] = backend_kwarg(default=None) + optim_params: dict[str, JsonValue] | None = backend_kwarg(default=None) class LrStepRequest(AdapterRequest): @@ -62,7 +62,7 @@ class LrStepRequest(AdapterRequest): # ``OptimizerParamScheduler.step(increment=...)``; the transformers scheduler has # no equivalent knob. - increment: Optional[int] = backend_only('megatron', default=None, ge=0) + increment: int | None = backend_only('megatron', default=None, ge=0) class ClipGradNormRequest(AdapterRequest): @@ -80,7 +80,7 @@ class ClipGradNormRequest(AdapterRequest): class ClipGradAndStepRequest(ClipGradNormRequest): """Body of ``POST /twinkle/clip_grad_and_step``.""" - optim_params: Optional[Dict[str, JsonValue]] = backend_kwarg(default=None) + optim_params: dict[str, JsonValue] | None = backend_kwarg(default=None) class CalculateMetricRequest(StrictRequest): @@ -109,26 +109,26 @@ class _InlineForwardBase(StrictRequest): """Fields common to the three inline forward endpoints.""" inputs: WireInputBatch - task: Optional[str] = backend_kwarg(default=None) - temperature: Optional[float] = backend_kwarg(default=None, gt=0) - return_logits: Optional[bool] = backend_kwarg(default=None) - micro_batch_size: Optional[int] = backend_kwarg(default=None, ge=1) - gradient_accumulation_steps: Optional[int] = backend_kwarg(default=None, ge=1) + task: str | None = backend_kwarg(default=None) + temperature: float | None = backend_kwarg(default=None, gt=0) + return_logits: bool | None = backend_kwarg(default=None) + micro_batch_size: int | None = backend_kwarg(default=None, ge=1) + gradient_accumulation_steps: int | None = backend_kwarg(default=None, ge=1) # Read only by the transformers backend. - sampling_masks: Optional[JsonValue] = backend_only('transformers', default=None) - router_replay_action: Optional[str] = backend_only('transformers', default=None) + sampling_masks: JsonValue | None = backend_only('transformers', default=None) + router_replay_action: str | None = backend_only('transformers', default=None) # Loss inputs (``advantages`` / ``old_logps`` / ``ref_outputs`` / ...). Their key # set is decided by the configured Loss, so they get a declared dict rather than # top-level fields; the flattening in ``backend_kwargs`` keeps the backend call # shape identical to before. - loss_kwargs: Dict[str, JsonValue] = passthrough() + loss_kwargs: dict[str, JsonValue] = passthrough() class ForwardRequest(_InlineForwardBase): """Body of ``POST /twinkle/forward``: keeps the graph, mutates no gradients.""" adapter_name: str - disable_lora: Optional[bool] = backend_kwarg(default=None) + disable_lora: bool | None = backend_kwarg(default=None) class ForwardOnlyRequest(_InlineForwardBase): @@ -139,17 +139,17 @@ class ForwardOnlyRequest(_InlineForwardBase): about. """ - adapter_name: Optional[str] = None - disable_lora: Optional[bool] = backend_kwarg(default=None) + adapter_name: str | None = None + disable_lora: bool | None = backend_kwarg(default=None) class ForwardBackwardTaskRequest(_InlineForwardBase): """Body of ``POST /twinkle/forward_backward``: accumulates gradients.""" adapter_name: str - seq_id: Optional[int] = None - sync_gradients: Optional[bool] = backend_kwarg(default=None) - loss_scale: Optional[float] = backend_kwarg(default=None) + seq_id: int | None = None + sync_gradients: bool | None = backend_kwarg(default=None) + loss_scale: float | None = backend_kwarg(default=None) # --------------------------------------------------------------------------- # @@ -164,30 +164,30 @@ class ForwardBackwardTaskRequest(_InlineForwardBase): class DataPlaneForwardRequest(StrictRequest): """Body of the ``*_from_data_plane`` forward endpoints.""" - input_refs: List[DataRef] = Field(min_length=1) - input_field: Optional[str] = None + input_refs: list[DataRef] = Field(min_length=1) + input_field: str | None = None # Values are *field paths*, not parameter values, so this is not a passthrough # region: nothing in it is forwarded verbatim. - kwarg_fields: Dict[str, str] = Field(default_factory=dict) + kwarg_fields: dict[str, str] = Field(default_factory=dict) adapter_name: str - seq_id: Optional[int] = None - task: Optional[str] = backend_kwarg(default=None) - temperature: Optional[float] = backend_kwarg(default=None, gt=0) - return_logits: Optional[bool] = backend_kwarg(default=None) - disable_lora: Optional[bool] = backend_kwarg(default=None) - micro_batch_size: Optional[int] = backend_kwarg(default=None, ge=1) - gradient_accumulation_steps: Optional[int] = backend_kwarg(default=None, ge=1) - loss_kwargs: Dict[str, JsonValue] = passthrough() + seq_id: int | None = None + task: str | None = backend_kwarg(default=None) + temperature: float | None = backend_kwarg(default=None, gt=0) + return_logits: bool | None = backend_kwarg(default=None) + disable_lora: bool | None = backend_kwarg(default=None) + micro_batch_size: int | None = backend_kwarg(default=None, ge=1) + gradient_accumulation_steps: int | None = backend_kwarg(default=None, ge=1) + loss_kwargs: dict[str, JsonValue] = passthrough() class DataPlaneForwardOnlyRequest(DataPlaneForwardRequest): """Body of ``POST /twinkle/forward_only_from_data_plane``.""" - output_ref: Optional[DataRef] = None - output_fields: Dict[str, str] = Field(default_factory=dict) + output_ref: DataRef | None = None + output_fields: dict[str, str] = Field(default_factory=dict) @model_validator(mode='after') - def validate_output(self) -> 'DataPlaneForwardOnlyRequest': + def validate_output(self) -> DataPlaneForwardOnlyRequest: if (self.output_ref is None) != (len(self.output_fields) == 0): raise ValueError('output_ref and output_fields must be configured together') return self @@ -209,19 +209,19 @@ def validate_output(self) -> 'DataPlaneForwardOnlyRequest': class SetLossRequest(StrictRequest): loss_cls: str adapter_name: str - init_kwargs: Dict[str, JsonValue] = passthrough() + init_kwargs: dict[str, JsonValue] = passthrough() class SetOptimizerRequest(StrictRequest): optimizer_cls: str adapter_name: str - init_kwargs: Dict[str, JsonValue] = passthrough() + init_kwargs: dict[str, JsonValue] = passthrough() class SetLrSchedulerRequest(StrictRequest): scheduler_cls: str adapter_name: str - init_kwargs: Dict[str, JsonValue] = passthrough() + init_kwargs: dict[str, JsonValue] = passthrough() class SetTemplateRequest(StrictRequest): @@ -235,26 +235,26 @@ class SetTemplateRequest(StrictRequest): template_cls: str adapter_name: str - init_kwargs: Dict[str, JsonValue] = passthrough() + init_kwargs: dict[str, JsonValue] = passthrough() class SetProcessorRequest(StrictRequest): processor_cls: str adapter_name: str - init_kwargs: Dict[str, JsonValue] = passthrough() + init_kwargs: dict[str, JsonValue] = passthrough() class AddMetricRequest(StrictRequest): metric_cls: str adapter_name: str - is_training: Optional[bool] = None - init_kwargs: Dict[str, JsonValue] = passthrough() + is_training: bool | None = None + init_kwargs: dict[str, JsonValue] = passthrough() class ApplyPatchRequest(StrictRequest): patch_cls: str adapter_name: str - init_kwargs: Dict[str, JsonValue] = passthrough() + init_kwargs: dict[str, JsonValue] = passthrough() # --------------------------------------------------------------------------- # @@ -264,20 +264,20 @@ class ApplyPatchRequest(StrictRequest): class SaveRequest(StrictRequest): adapter_name: str - name: Optional[str] = None + name: str | None = None save_optimizer: bool = False is_sampler: bool = False # If True, delete existing sampler weights before saving - consumed_train_samples: Optional[int] = backend_kwarg(default=None, ge=0) - merge_lora: Optional[bool] = backend_only('megatron', default=None) + consumed_train_samples: int | None = backend_kwarg(default=None, ge=0) + merge_lora: bool | None = backend_only('megatron', default=None) class LoadRequest(StrictRequest): adapter_name: str name: str load_optimizer: bool = False - no_load_optim: Optional[bool] = backend_only('megatron', default=None) - no_load_rng: Optional[bool] = backend_only('megatron', default=None) - strict: Optional[bool] = backend_only('transformers', default=None) + no_load_optim: bool | None = backend_only('megatron', default=None) + no_load_rng: bool | None = backend_only('megatron', default=None) + strict: bool | None = backend_only('transformers', default=None) class ResumeFromCheckpointRequest(StrictRequest): @@ -292,10 +292,10 @@ class AddAdapterRequest(StrictRequest): adapter_name: str # ``config`` is None for full-parameter training (no LoRA adapter) and a # serialized LoraConfig string for LoRA training. - config: Optional[str] = None - save_dir: Optional[str] = None - gradient_accumulation_steps: Optional[int] = backend_kwarg(default=None, ge=1) - init_kwargs: Dict[str, JsonValue] = passthrough() + config: str | None = None + save_dir: str | None = None + gradient_accumulation_steps: int | None = backend_kwarg(default=None, ge=1) + init_kwargs: dict[str, JsonValue] = passthrough() class UploadToHubRequest(StrictRequest): @@ -305,9 +305,9 @@ class UploadToHubRequest(StrictRequest): the client waits through the future layer, so the flag could only ever be ignored. """ - checkpoint_dir: Union[str, Dict[str, Any]] + checkpoint_dir: str | dict[str, Any] hub_model_id: str - hub_token: Optional[str] = None + hub_token: str | None = None @field_validator('checkpoint_dir', mode='before') @classmethod @@ -369,18 +369,18 @@ class GetTrainConfigsResponse(ResponseModel): class CalculateMetricResponse(ResponseModel): """Response for /calculate_metric endpoint (returns Dict).""" - result: Dict[str, Any] + result: dict[str, Any] class SaveResponse(ResponseModel): """Response for /save endpoint (returns twinkle path + checkpoint dir).""" twinkle_path: str - checkpoint_dir: Optional[str] = None + checkpoint_dir: str | None = None class TrainingProgressResponse(ResponseModel): """Response for /resume_from_checkpoint endpoint.""" - result: Dict[str, Any] + result: dict[str, Any] # --- Void responses (return None → OkResponse) --- diff --git a/src/twinkle_client/types/processor.py b/src/twinkle_client/types/processor.py index 7fdb1a64..f778f7ec 100644 --- a/src/twinkle_client/types/processor.py +++ b/src/twinkle_client/types/processor.py @@ -27,7 +27,7 @@ class ProcessorCreateRequest(StrictRequest): processor_type: str class_type: str - init_kwargs: Dict[str, JsonValue] = passthrough() + init_kwargs: dict[str, JsonValue] = passthrough() class ProcessorHeartbeatRequest(StrictRequest): @@ -37,7 +37,7 @@ class ProcessorHeartbeatRequest(StrictRequest): class ProcessorCallRequest(StrictRequest): processor_id: str function: str - call_kwargs: Dict[str, JsonValue] = passthrough() + call_kwargs: dict[str, JsonValue] = passthrough() class ProcessorCreateResponse(ResponseModel): diff --git a/src/twinkle_client/types/sampler.py b/src/twinkle_client/types/sampler.py index 5e5207c1..d69aa649 100644 --- a/src/twinkle_client/types/sampler.py +++ b/src/twinkle_client/types/sampler.py @@ -36,43 +36,40 @@ class SampleRequest(StrictRequest): """ inputs: WireInputBatch = Field(..., description='Trajectory or InputFeature entries to sample from') - sampling_params: Optional[Dict[str, JsonValue]] = Field( + sampling_params: dict[str, JsonValue] | None = Field( None, description='Sampling parameters (max_tokens, temperature, num_samples, etc.)') adapter_name: str = Field('', description='Adapter name for LoRA inference') - adapter_uri: Optional[str] = Field( - None, description='Adapter URI (twinkle:// path or local path) for LoRA inference') + adapter_uri: str | None = Field(None, description='Adapter URI (twinkle:// path or local path) for LoRA inference') class SampledSequenceModel(ResponseModel): """A single sampled sequence, mirroring twinkle.data_format.SampledSequence.""" stop_reason: StopReason = Field(..., description="Stop reason: 'length' or 'stop'") - tokens: List[int] = Field(..., description='Token IDs of the sampled sequence') - logprobs: Optional[List[Optional[List[Tuple[int, float]]]]] = Field(None, description='Per-token log-probabilities') - decoded: Optional[str] = Field(None, description='Decoded text of the sampled sequence') - new_input_feature: Optional[Dict[str, Any]] = Field( + tokens: list[int] = Field(..., description='Token IDs of the sampled sequence') + logprobs: list[list[tuple[int, float]] | None] | None = Field(None, description='Per-token log-probabilities') + decoded: str | None = Field(None, description='Decoded text of the sampled sequence') + new_input_feature: dict[str, Any] | None = Field( None, description='Updated InputFeature after sampling (input_ids, labels, etc.)') class SampleResponseModel(ResponseModel): """Mirroring twinkle.data_format.SampleResponse.""" - sequences: List[SampledSequenceModel] = Field( - ..., description='List of sampled sequences') - prompt_token_ids: Optional[List[int]] = Field( - None, description='Token IDs of the prompt the sequences continue') - prompt_logprobs: Optional[List[Optional[float]]] = None - topk_prompt_logprobs: Optional[List[Optional[List[Tuple[int, float]]]]] = None + sequences: list[SampledSequenceModel] = Field(..., description='List of sampled sequences') + prompt_token_ids: list[int] | None = Field(None, description='Token IDs of the prompt the sequences continue') + prompt_logprobs: list[float | None] | None = None + topk_prompt_logprobs: list[list[tuple[int, float]] | None] | None = None class SampleResponseModelList(ResponseModel): """Response body for the /sample endpoint""" - samples: List[SampleResponseModel] = Field(..., description='List of sample responses') + samples: list[SampleResponseModel] = Field(..., description='List of sample responses') class SamplerSetTemplateRequest(StrictRequest): """Request body for the sampler ``/set_template`` endpoint.""" template_cls: str = Field(..., description="Template class name (e.g. 'Template')") adapter_name: str = Field('', description='Adapter name to associate the template with') - init_kwargs: Dict[str, JsonValue] = passthrough() + init_kwargs: dict[str, JsonValue] = passthrough() class SamplerSetTemplateResponse(ResponseModel): diff --git a/src/twinkle_client/utils/patch_tinker.py b/src/twinkle_client/utils/patch_tinker.py index ed9abaa8..352db28f 100644 --- a/src/twinkle_client/utils/patch_tinker.py +++ b/src/twinkle_client/utils/patch_tinker.py @@ -57,9 +57,8 @@ def _patched_async_tinker_init( if api_key is None: api_key = os.environ.get('TWINKLE_SERVER_TOKEN') if api_key is None: - raise TinkerError( - 'The api_key client option must be set either by passing api_key to the client or by setting the TWINKLE_SERVER_TOKEN environment variable' - ) + raise TinkerError('The api_key client option must be set either by passing api_key to the client or by ' + 'setting the TWINKLE_SERVER_TOKEN environment variable') # REMOVED: api_key 'tml-' prefix validation # Original code: # if not api_key.startswith("tml-"): @@ -120,6 +119,7 @@ def _patched_from_tinker_path(cls, tinker_path: str) -> Any: def _make_patched_service_client_init(original): + def _patched_service_client_init(self, user_metadata=None, **kwargs): """Patched version of ServiceClient.__init__ that injects Twinkle-specific headers.""" # Resolve api_key with the same priority order used by AsyncTinker: @@ -147,8 +147,8 @@ def _create_full_training_client_submit(self, base_model, seed=None, user_metada the training loop (forward_backward / optim_step / save_weights / ...) is identical to the LoRA path. """ - from tinker.lib.public_interfaces import service_client as _sc from tinker.lib.internal_client_holder import ClientConnectionPoolType + from tinker.lib.public_interfaces import service_client as _sc session_id = self.holder.get_session_id() model_seq_id = self.holder.get_training_client_id() @@ -184,8 +184,7 @@ async def _create_full_training_client_async(): def _create_full_training_client(self, base_model, seed=None, user_metadata=None): """Create a full-parameter (non-LoRA) training client (blocking).""" - return _create_full_training_client_submit( - self, base_model, seed=seed, user_metadata=user_metadata).result() + return _create_full_training_client_submit(self, base_model, seed=seed, user_metadata=user_metadata).result() async def _create_full_training_client_async(self, base_model, seed=None, user_metadata=None): From f7dc64ed1bfb7f14b85415fedd0ae99c655d1e7e Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Fri, 18 Sep 2026 07:34:56 +0000 Subject: [PATCH 21/32] refactor(client,server): module renames, agent-tools split, and rollout cleanup Server: - Rename utils/validation.py -> utils/auth.py to end the collision with the server/validation preflight package (auth/session helpers vs request checks). - Rename model/utils.py -> model/data_plane_inputs.py to name what it does. Client: - Rename http/http_utils.py -> http/client.py and http/utils.py -> http/context.py; update all importers. - Split auto/agent/tools.py into tool_schemas.py (schemas), server_tools.py and search_tools.py (ToolExecutor mixins); add test_auto_agent_tools.py. - Rework rollout/multi_turn.py into an explicit _RolloutState with _initialize_state / _process_sequence helpers. All touched files pass `pre-commit run --all-files` (client now linted). --- src/twinkle/server/common/router.py | 4 +- src/twinkle/server/deployment.py | 2 +- src/twinkle/server/gateway/tinker_handlers.py | 2 +- .../server/gateway/twinkle_handlers.py | 2 +- src/twinkle/server/lifecycle/submit.py | 2 +- src/twinkle/server/model/app.py | 2 +- .../model/{utils.py => data_plane_inputs.py} | 0 src/twinkle/server/model/twinkle_handlers.py | 6 +- .../server/processor/twinkle_handlers.py | 2 +- src/twinkle/server/sampler/app.py | 2 +- src/twinkle/server/telemetry/tracing.py | 1 - .../server/utils/{validation.py => auth.py} | 0 src/twinkle_client/_future.py | 2 +- src/twinkle_client/auto/agent/search_tools.py | 63 + src/twinkle_client/auto/agent/server_tools.py | 803 +++++++++++ src/twinkle_client/auto/agent/tool_schemas.py | 338 +++++ src/twinkle_client/auto/agent/tools.py | 1194 +---------------- src/twinkle_client/common/serialize.py | 11 +- src/twinkle_client/http/__init__.py | 6 +- .../http/{http_utils.py => client.py} | 2 +- .../http/{utils.py => context.py} | 0 src/twinkle_client/rollout/multi_turn.py | 286 ++-- src/twinkle_client/types/data.py | 16 +- src/twinkle_client/types/session.py | 2 +- src/twinkle_client/utils/patch_tinker.py | 6 +- tests/server/lifecycle/test_static_guards.py | 12 +- .../server/model/test_twinkle_async_inputs.py | 33 +- tests/twinkle_client/test_async_components.py | 33 +- tests/twinkle_client/test_async_rl_workers.py | 2 +- tests/twinkle_client/test_auto_agent_tools.py | 69 + .../test_client_multi_turn_rollout.py | 49 +- .../test_client_orchestrated_grpo.py | 35 +- tests/twinkle_client/test_data_plane_async.py | 19 +- tests/twinkle_client/test_error_parsing.py | 4 +- tests/twinkle_client/test_future_layer.py | 11 +- tests/twinkle_client/test_request_builder.py | 22 +- tests/twinkle_client/test_types_contract.py | 15 +- 37 files changed, 1592 insertions(+), 1466 deletions(-) rename src/twinkle/server/model/{utils.py => data_plane_inputs.py} (100%) rename src/twinkle/server/utils/{validation.py => auth.py} (100%) create mode 100644 src/twinkle_client/auto/agent/search_tools.py create mode 100644 src/twinkle_client/auto/agent/server_tools.py create mode 100644 src/twinkle_client/auto/agent/tool_schemas.py rename src/twinkle_client/http/{http_utils.py => client.py} (98%) rename src/twinkle_client/http/{utils.py => context.py} (100%) create mode 100644 tests/twinkle_client/test_auto_agent_tools.py diff --git a/src/twinkle/server/common/router.py b/src/twinkle/server/common/router.py index e0ccef06..0aabc63b 100644 --- a/src/twinkle/server/common/router.py +++ b/src/twinkle/server/common/router.py @@ -1,8 +1,6 @@ # Copyright (c) ModelScope Contributors. All rights reserved. # Moved from tinker/common/router.py — logic unchanged. -from ray.serve.request_router import (FIFOMixin, MultiplexMixin, PendingRequest, ReplicaID, ReplicaResult, - RequestRouter, RunningReplica) -from typing import Dict, List, Optional +from ray.serve.request_router import FIFOMixin, MultiplexMixin, PendingRequest, ReplicaID, RequestRouter, RunningReplica from twinkle.server.state import ServerState, get_server_state from twinkle.utils.logger import get_logger diff --git a/src/twinkle/server/deployment.py b/src/twinkle/server/deployment.py index d795d31b..239686bf 100644 --- a/src/twinkle/server/deployment.py +++ b/src/twinkle/server/deployment.py @@ -38,7 +38,7 @@ from twinkle.server.exceptions import TwinkleServerError from twinkle.server.telemetry.middleware import create_metrics_middleware from twinkle.server.telemetry.tracing import create_tracing_middleware -from twinkle.server.utils.validation import verify_request_token +from twinkle.server.utils.auth import verify_request_token from twinkle.server.validation.errors import register_validation_error_handler from twinkle.utils.logger import get_logger from twinkle_client.types.errors import ErrorCategory, ErrorPayload diff --git a/src/twinkle/server/gateway/tinker_handlers.py b/src/twinkle/server/gateway/tinker_handlers.py index 268cd316..19c0f697 100644 --- a/src/twinkle/server/gateway/tinker_handlers.py +++ b/src/twinkle/server/gateway/tinker_handlers.py @@ -19,8 +19,8 @@ from twinkle.hub import HubOperation from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager from twinkle.server.lifecycle.poll_config import long_poll_window, retrieve_poll_interval +from twinkle.server.utils.auth import get_token_from_request from twinkle.server.utils.task_errors import error_payload_from_stored -from twinkle.server.utils.validation import get_token_from_request from twinkle.utils.logger import get_logger logger = get_logger() diff --git a/src/twinkle/server/gateway/twinkle_handlers.py b/src/twinkle/server/gateway/twinkle_handlers.py index 396fb74c..b0b5366a 100644 --- a/src/twinkle/server/gateway/twinkle_handlers.py +++ b/src/twinkle/server/gateway/twinkle_handlers.py @@ -19,7 +19,7 @@ from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager, validate_user_path from twinkle.server.lifecycle.envelope import envelope_from_record from twinkle.server.lifecycle.poll_config import long_poll_window, retrieve_poll_interval -from twinkle.server.utils.validation import get_token_from_request +from twinkle.server.utils.auth import get_token_from_request from twinkle.utils.logger import get_logger from twinkle_client.types.lifecycle import TERMINAL_STATUSES diff --git a/src/twinkle/server/lifecycle/submit.py b/src/twinkle/server/lifecycle/submit.py index fe2dd4ff..dc02074e 100644 --- a/src/twinkle/server/lifecycle/submit.py +++ b/src/twinkle/server/lifecycle/submit.py @@ -13,7 +13,7 @@ from typing import Any from twinkle.data_format import InputFeature, Trajectory, is_encoded -from twinkle.server.utils.validation import get_session_id_from_request +from twinkle.server.utils.auth import get_session_id_from_request from twinkle.server.validation import assert_request_supported from twinkle_client.types.base import FieldRole, fields_with_role from twinkle_client.types.data import export_batch diff --git a/src/twinkle/server/model/app.py b/src/twinkle/server/model/app.py index 42c4d799..dd7c98b2 100644 --- a/src/twinkle/server/model/app.py +++ b/src/twinkle/server/model/app.py @@ -19,10 +19,10 @@ from twinkle.server.exceptions import FullModeBusyError from twinkle.server.state import ServerState, get_server_state from twinkle.server.utils import wrap_builder_with_device_group_env +from twinkle.server.utils.auth import get_token_from_request from twinkle.server.utils.backend_dispatch import BackendSelector from twinkle.server.utils.session_resource import AdapterManagerMixin from twinkle.server.utils.task_queue import TaskQueueConfig, TaskQueueMixin -from twinkle.server.utils.validation import get_token_from_request from twinkle.utils.logger import get_logger from .tinker_handlers import _register_tinker_routes from .twinkle_handlers import _register_twinkle_routes diff --git a/src/twinkle/server/model/utils.py b/src/twinkle/server/model/data_plane_inputs.py similarity index 100% rename from src/twinkle/server/model/utils.py rename to src/twinkle/server/model/data_plane_inputs.py diff --git a/src/twinkle/server/model/twinkle_handlers.py b/src/twinkle/server/model/twinkle_handlers.py index 8495c745..c20665f5 100644 --- a/src/twinkle/server/model/twinkle_handlers.py +++ b/src/twinkle/server/model/twinkle_handlers.py @@ -25,9 +25,9 @@ from twinkle.server.exceptions import RequestRejectedError, TrainModeMismatchError from twinkle.server.lifecycle.submit import (backend_kwargs, input_metrics, resolve_twinkle_adapter_name, run_submit, to_backend_inputs) -from twinkle.server.model.utils import (data_plane_request_shape, merge_forward_kwargs, resolve_data_plane_model_inputs, - select_output_rows) -from twinkle.server.utils.validation import get_session_id_from_request +from twinkle.server.model.data_plane_inputs import (data_plane_request_shape, merge_forward_kwargs, + resolve_data_plane_model_inputs, select_output_rows) +from twinkle.server.utils.auth import get_session_id_from_request from twinkle.server.validation import BackendCapability from twinkle.utils.logger import get_logger diff --git a/src/twinkle/server/processor/twinkle_handlers.py b/src/twinkle/server/processor/twinkle_handlers.py index 58fd52d2..201fd6e2 100644 --- a/src/twinkle/server/processor/twinkle_handlers.py +++ b/src/twinkle/server/processor/twinkle_handlers.py @@ -21,7 +21,7 @@ import twinkle_client.types as types from twinkle.server.telemetry.correlation import SESSION_ID, TOKEN_ID from twinkle.server.telemetry.tracing import traced_operation -from twinkle.server.utils.validation import get_session_id_from_request, get_token_from_request +from twinkle.server.utils.auth import get_session_id_from_request, get_token_from_request from twinkle.utils.logger import get_logger logger = get_logger() diff --git a/src/twinkle/server/sampler/app.py b/src/twinkle/server/sampler/app.py index 3e7d0145..d38e6b0f 100644 --- a/src/twinkle/server/sampler/app.py +++ b/src/twinkle/server/sampler/app.py @@ -15,9 +15,9 @@ from twinkle.server.deployment import LazyCleanupMixin, bind_deployment, build_deployment_app, init_twinkle_runtime from twinkle.server.state import ServerState, get_server_state from twinkle.server.utils import wrap_builder_with_device_group_env +from twinkle.server.utils.auth import get_token_from_request from twinkle.server.utils.backend_dispatch import BackendSelector from twinkle.server.utils.task_queue import TaskQueueConfig, TaskQueueMixin -from twinkle.server.utils.validation import get_token_from_request from twinkle.utils.logger import get_logger from .tinker_handlers import _register_tinker_sampler_routes from .twinkle_handlers import _register_twinkle_sampler_routes diff --git a/src/twinkle/server/telemetry/tracing.py b/src/twinkle/server/telemetry/tracing.py index 4f3b6473..18e351cf 100644 --- a/src/twinkle/server/telemetry/tracing.py +++ b/src/twinkle/server/telemetry/tracing.py @@ -9,7 +9,6 @@ try: from opentelemetry import trace - from opentelemetry.context import Context from opentelemetry.propagate import extract, inject _OTEL_AVAILABLE = True except Exception: diff --git a/src/twinkle/server/utils/validation.py b/src/twinkle/server/utils/auth.py similarity index 100% rename from src/twinkle/server/utils/validation.py rename to src/twinkle/server/utils/auth.py diff --git a/src/twinkle_client/_future.py b/src/twinkle_client/_future.py index 318a3ac2..493b5e28 100644 --- a/src/twinkle_client/_future.py +++ b/src/twinkle_client/_future.py @@ -14,7 +14,7 @@ from twinkle_client.exceptions import TaskCancelledError, TaskFailedError, TaskRecordLostError, TaskWaitTimeoutError from twinkle_client.http import http_post -from twinkle_client.http.utils import get_base_url +from twinkle_client.http.context import get_base_url from twinkle_client.types.lifecycle import TERMINAL_STATUSES, TaskEnvelope logger = logging.getLogger('twinkle_client') diff --git a/src/twinkle_client/auto/agent/search_tools.py b/src/twinkle_client/auto/agent/search_tools.py new file mode 100644 index 00000000..4bbb652d --- /dev/null +++ b/src/twinkle_client/auto/agent/search_tools.py @@ -0,0 +1,63 @@ +# Copyright (c) Twinkle Contributors. All rights reserved. +"""Private ModelScope Hub search tools used by ``ToolExecutor``.""" +from __future__ import annotations + +import asyncio + + +class _SearchTools: + + async def _tool_search_datasets(self, query: str, limit: int = 5) -> dict: + """Search ModelScope for datasets.""" + return await self._search_hub('datasets', query, limit) + + async def _tool_search_models(self, query: str, limit: int = 5) -> dict: + """Search ModelScope for models.""" + return await self._search_hub('models', query, limit) + + async def _search_hub(self, resource_type: str, query: str, limit: int) -> dict: + """Unified ModelScope Hub search for models or datasets.""" + + def _search(): + if resource_type == 'datasets': + return self._search_datasets_impl(query, limit) + else: + return self._search_models_impl(query, limit) + + try: + items = await asyncio.get_event_loop().run_in_executor(None, _search) + return {'query': query, 'results': items} + except Exception as e: + return {'error': f'{resource_type.title()} search failed: {e}'} + + @staticmethod + def _search_datasets_impl(query: str, limit: int) -> list[dict]: + """Search datasets via ModelScope SDK (new API).""" + from modelscope.hub.api import HubApi + api = HubApi() + result = api.list_datasets('', search=query, page_size=limit) + datasets = result.get('datasets', []) + return [{'id': d.get('id', ''), 'name': d.get('display_name', d.get('id', ''))} for d in datasets] + + @staticmethod + def _search_models_impl(query: str, limit: int) -> list[dict]: + """Search models via ModelScope HTTP API (SDK doesn't support search).""" + import requests + resp = requests.put( + 'https://modelscope.cn/api/v1/models/', + json={ + 'Name': query, + 'PageSize': limit, + 'PageNumber': 1 + }, + timeout=15, + ) + resp.raise_for_status() + data = resp.json() + if not data.get('Success'): + raise RuntimeError(data.get('Message', 'Unknown error')) + models = data.get('Data', {}).get('Models', []) + return [{ + 'id': f"{m.get('Path', '')}/{m.get('Name', '')}", + 'name': m.get('ChineseName') or m.get('Name', ''), + } for m in models] diff --git a/src/twinkle_client/auto/agent/server_tools.py b/src/twinkle_client/auto/agent/server_tools.py new file mode 100644 index 00000000..d73f5c94 --- /dev/null +++ b/src/twinkle_client/auto/agent/server_tools.py @@ -0,0 +1,803 @@ +# Copyright (c) Twinkle Contributors. All rights reserved. +"""Private server lifecycle and cluster tools used by ``ToolExecutor``.""" +from __future__ import annotations + +import asyncio +import json +import os + + +class _ServerTools: + """Server-side operations mixed into ``ToolExecutor``. + + ``ToolExecutor`` owns the URL value; declaring it here makes that host + requirement visible without assigning runtime state in the mixin. + """ + + _server_url: str | None + + async def _check_server_health(self, url: str) -> bool: + """Check if Twinkle Server is reachable (non-blocking).""" + import urllib.error + import urllib.request + + def _probe(): + try: + req = urllib.request.Request(f'{url}/api/v1/healthz', method='GET') + urllib.request.urlopen(req, timeout=3) + return True + except (urllib.error.URLError, OSError): + # Try a simpler connectivity check + try: + urllib.request.urlopen(url, timeout=3) + return True + except (urllib.error.URLError, OSError): + return False + + return await asyncio.get_event_loop().run_in_executor(None, _probe) + + async def _tool_start_server( + self, + model_id: str, + train_gpus: int | None = None, + port: int = 8000, + backend: str = 'transformers', + samplers: list[dict] | None = None, + ) -> dict: + """Start Ray cluster + Twinkle Server. Idempotent. Supports multi-model.""" + server_url = self._server_url or os.environ.get('TWINKLE_SERVER_URL') or f'http://localhost:{port}' + + # Idempotent: skip if already running + if await self._check_server_health(server_url): + self._server_url = server_url + return {'status': 'already_running', 'server_url': server_url} + + def _start(): + sampler_list = samplers or [] + + # Step 1: Detect hardware & compute GPU partition + total_hw_gpus = self._detect_gpu_count() + if total_hw_gpus == 0: + return {'status': 'error', 'error': 'No GPUs detected. Cannot start training server.'} + + alloc = self._compute_gpu_allocation(sampler_list, train_gpus, total_hw_gpus) + if 'error' in alloc: + return {'status': 'error', 'error': alloc['error']} + t_gpus, sampler_gpu_total = alloc['train_gpus'], alloc['sampler_gpus'] + + # Step 2: Generate server_config.yaml + config_path = self._generate_server_config( + model_id=model_id, + train_gpus=t_gpus, + port=port, + backend=backend, + samplers=sampler_list, + ) + + # Step 3: Start Ray cluster (multi-node GPU partitioning) + ray_err = self._start_ray_cluster(t_gpus, sampler_gpu_total) + if ray_err: + return {'status': 'error', 'error': ray_err} + + # Step 4: Launch Twinkle Server process + proc, log_path, err = self._launch_server_process(config_path) + if err: + return {'status': 'error', 'error': err} + + # Step 5: Wait for readiness (healthz + sampler engine) + return self._wait_server_ready( + server_url=server_url, + proc=proc, + log_path=log_path, + sampler_list=sampler_list, + model_id=model_id, + t_gpus=t_gpus, + backend=backend, + config_path=config_path, + ) + + result = await asyncio.get_event_loop().run_in_executor(None, _start) + if result.get('status') in ('started', 'already_running'): + self._server_url = server_url + return result + + @staticmethod + def _detect_gpu_count() -> int: + """Detect total hardware GPU count via nvidia-smi.""" + import subprocess as _sp + try: + r = _sp.run( + ['nvidia-smi', '--query-gpu=index', '--format=csv,noheader'], + capture_output=True, + text=True, + timeout=10, + ) + if r.returncode == 0: + return len([ln for ln in r.stdout.strip().split('\n') if ln.strip()]) + except (FileNotFoundError, OSError): + pass + return 0 + + @staticmethod + def _compute_gpu_allocation( + sampler_list: list[dict], + train_gpus: int | None, + total_hw_gpus: int, + ) -> dict: + """Compute GPU partition: {train_gpus, sampler_gpus} or {error}.""" + sampler_gpu_total = 0 + for s in sampler_list: + s_tp = s.get('tp', 1) + s_dp, s_gpus = s.get('dp'), s.get('gpus') + if s_gpus is not None: + sampler_gpu_total += s_gpus + elif s_dp is not None: + sampler_gpu_total += s_tp * s_dp + else: + sampler_gpu_total += s_tp # default dp=1 + + t_gpus = train_gpus if train_gpus is not None else max(1, total_hw_gpus - sampler_gpu_total) + needed = t_gpus + sampler_gpu_total + if needed > total_hw_gpus: + return { + 'error': (f'Requested {needed} GPUs (train={t_gpus}, samplers={sampler_gpu_total}) ' + f'but only {total_hw_gpus} available.'), + } + return {'train_gpus': t_gpus, 'sampler_gpus': sampler_gpu_total} + + @staticmethod + def _start_ray_cluster(train_gpus: int, sampler_gpus: int) -> str | None: + """Start Ray multi-node cluster with GPU partitioning. + + Each role gets its own Ray node with dedicated CUDA_VISIBLE_DEVICES + so GPUs are indexed from 0 within each node. This prevents the + GPU ID mapping issues that occur with a single-node setup. + + On a single machine, multiple raylets need separate --temp-dir to + avoid being detected as "already running". + + Returns an error message on failure, or None on success. + """ + import subprocess as _sp + import tempfile + from pathlib import Path + + _sp.run(['ray', 'stop', '--force'], capture_output=True, timeout=15) + + # Create unique temp dirs so each `ray start` spawns a separate raylet + ray_base = Path(tempfile.gettempdir()) / 'twinkle_ray' + ray_base.mkdir(parents=True, exist_ok=True) + + def _ray_node( + devices: str, + num_gpus: int, + *, + head: bool = False, + node_name: str = 'worker', + ) -> str | None: + env = os.environ.copy() + env['CUDA_VISIBLE_DEVICES'] = devices + temp_dir = str(ray_base / node_name) + cmd = ['ray', 'start', f'--temp-dir={temp_dir}'] + if head: + cmd += ['--head', '--port=6379', '--disable-usage-stats', '--include-dashboard=false'] + else: + cmd += ['--address=127.0.0.1:6379'] + cmd.append(f'--num-gpus={num_gpus}') + r = _sp.run(cmd, capture_output=True, text=True, timeout=30, env=env) + if r.returncode != 0 and 'already' not in r.stderr.lower(): + return r.stderr.strip() + return None + + # Head node — training model GPUs + model_devices = ','.join(str(i) for i in range(train_gpus)) + err = _ray_node(model_devices, train_gpus, head=True, node_name='head') + if err: + return f'Ray head start failed: {err}' + + # GPU Worker node — sampler GPUs + if sampler_gpus > 0: + sampler_devices = ','.join(str(i) for i in range(train_gpus, train_gpus + sampler_gpus)) + err = _ray_node(sampler_devices, sampler_gpus, node_name='gpu_worker') + if err: + return f'Ray GPU worker start failed: {err}' + + # CPU Worker node — processor (no GPU) + _ray_node('', 0, node_name='cpu_worker') + return None + + @staticmethod + def _launch_server_process(config_path: str) -> tuple: + """Launch Twinkle Server as a detached background process. + + Returns (proc, log_path, error). On success error is None. + """ + import subprocess as _sp + from pathlib import Path + + log_dir = Path.home() / '.cache' / 'twinkle' + log_dir.mkdir(parents=True, exist_ok=True) + log_path = str(log_dir / 'server.log') + log_file = open(log_path, 'w') + + cmd = ['python', '-m', 'twinkle.server', 'launch', '--config', config_path] + try: + proc = _sp.Popen( + cmd, + stdout=log_file, + stderr=_sp.STDOUT, + start_new_session=True, + ) + except OSError as e: + log_file.close() + return None, log_path, f'Failed to start Twinkle server: {e}' + return proc, log_path, None + + @staticmethod + def _wait_server_ready( + server_url: str, + proc, + log_path: str, + sampler_list: list[dict], + model_id: str, + t_gpus: int, + backend: str, + config_path: str, + ) -> dict: + """Poll server until healthy (healthz + sampler engine ready).""" + import time + import urllib.error + import urllib.request + + timeout_s = 120 if sampler_list else 60 + needed = t_gpus + sum(s.get('gpus') or (s.get('tp', 1) * s.get('dp', 1)) for s in sampler_list) + + for _ in range(timeout_s): + time.sleep(1) + if proc.poll() is not None: + # Server died — read log tail to diagnose + log_tail = _ServerTools._read_log_tail(log_path, max_chars=2000) + error_msg = (f'Server exited immediately (code={proc.returncode}). ' + f'Model: {model_id}, GPUs: {t_gpus}, Samplers: {len(sampler_list)}.\n' + f'--- server.log tail ---\n{log_tail}') + return { + 'status': 'error', + 'error': error_msg, + 'log_path': log_path, + 'hint': 'Check if required packages are installed (pip install -e ".[all]").', + } + try: + urllib.request.urlopen(f'{server_url}/api/v1/healthz', timeout=2) + except (OSError, Exception): + continue + + # healthz OK — additionally wait for sampler vLLM engines + if sampler_list and not _ServerTools._probe_sampler_ready(server_url, sampler_list, model_id): + return { + 'status': 'started', + 'warning': 'Server is up but sampler may still be loading.', + 'server_url': server_url, + 'server_pid': proc.pid, + 'model_id': model_id, + 'log_path': log_path, + } + + return { + 'status': 'started', + 'server_url': server_url, + 'server_pid': proc.pid, + 'model_id': model_id, + 'train_gpus': t_gpus, + 'backend': backend, + 'samplers': [s.get('model_id') for s in sampler_list], + 'total_gpus_used': needed, + 'config_path': config_path, + 'log_path': log_path, + } + + return { + 'status': 'timeout', + 'error': 'Health check did not pass within timeout. Models may still be loading.', + 'server_pid': proc.pid, + 'log_path': log_path, + } + + @staticmethod + def _read_log_tail(log_path: str, max_chars: int = 2000) -> str: + """Read the tail of a log file for error diagnosis.""" + try: + with open(log_path, errors='replace') as f: + content = f.read() + if len(content) <= max_chars: + return content.strip() + return content[-max_chars:].strip() + except OSError: + return '(could not read log file)' + + @staticmethod + def _probe_sampler_ready(server_url: str, sampler_list: list[dict], fallback_model_id: str) -> bool: + """Probe sampler route up to 90s to confirm vLLM engine is loaded.""" + import time + import urllib.error + import urllib.request + + s_mid = sampler_list[0].get('model_id', fallback_model_id) + probe_url = f'{server_url}/api/v1/sampler/{s_mid}/twinkle/create' + + for _ in range(90): + try: + req = urllib.request.Request( + probe_url, + method='POST', + data=b'{}', + headers={'Content-Type': 'application/json'}, + ) + urllib.request.urlopen(req, timeout=5) + return True # non-error response = ready + except urllib.error.HTTPError as e: + if e.code < 500: + return True # 4xx = actor alive, just bad request + time.sleep(1) # 5xx = still loading + except (OSError, Exception): + time.sleep(1) + return False + + @staticmethod + def _generate_server_config( + model_id: str, + train_gpus: int, + port: int = 8000, + backend: str = 'transformers', + samplers: list[dict] | None = None, + ) -> str: + """Generate a server_config.yaml from template and return its path. + + Supports multi-model topology: + - 1 training model (student) + - N sampler/teacher models (for RL/OPD) + - 1 processor service + """ + import yaml + from pathlib import Path + + sampler_list = samplers or [] + + # Sanitize model name for use in route/names + def _short(mid: str) -> str: + return mid.split('/')[-1] if '/' in mid else mid + + model_short = _short(model_id) + + # Collect all model IDs for supported_models + all_model_ids = [model_id] + [s['model_id'] for s in sampler_list] + + # === Build applications list === + applications = [] + + # 1. API Gateway + applications.append({ + 'name': + 'server', + 'route_prefix': + '/api/v1', + 'import_path': + 'server', + 'args': { + 'server_config': { + 'per_token_model_limit': 3 + }, + 'supported_models': all_model_ids, + }, + 'deployments': [{ + 'name': 'TinkerCompatServer', + 'max_ongoing_requests': 50, + 'autoscaling_config': { + 'min_replicas': 1, + 'max_replicas': 1, + 'target_ongoing_requests': 128, + }, + 'ray_actor_options': { + 'num_cpus': 0.1 + }, + }], + }) + + # 2. Build GPU-requiring applications (model + samplers), + # then sort by GPU count DESCENDING before appending. + # Largest PG deploys first → it has the fewest node choices → + # avoids GPU scheduling deadlock on single-machine multi-node. + gpu_apps: list[tuple[int, dict]] = [] # (gpu_count, app_config) + + # 2a. Training model worker (student) + gpu_apps.append(( + train_gpus, + { + 'name': + f'models-{model_short}', + 'route_prefix': + f'/api/v1/model/{model_id}', + 'import_path': + 'model', + 'args': { + 'backend': backend, + 'model_id': f'ms://{model_id}', + 'max_length': 500000, # total tokens per forward pass (must match max_input_tokens) + 'nproc_per_node': train_gpus, + 'device_group': { + 'name': 'model', + 'ranks': train_gpus, + 'device_type': 'cuda', + }, + 'device_mesh': { + 'device_type': 'cuda', + 'dp_size': train_gpus, + }, + 'queue_config': { + 'rps_limit': 100, + 'tps_limit': 100000, + 'max_input_tokens': 500000, + }, + 'adapter_config': { + 'adapter_timeout': 600, + }, + }, + 'deployments': [{ + 'name': 'ModelManagement', + 'autoscaling_config': { + 'min_replicas': 1, + 'max_replicas': 1, + 'target_ongoing_requests': 16, + }, + 'ray_actor_options': { + 'num_cpus': 0.1, + 'runtime_env': { + 'env_vars': { + 'TWINKLE_TRUST_REMOTE_CODE': '1' + }, + }, + }, + }], + })) + + # 2b. Sampler/teacher models + sampler_name_count: dict[str, int] = {} + for sampler_cfg in sampler_list: + s_model_id = sampler_cfg['model_id'] + s_short = _short(s_model_id) + + # Deduplicate names when multiple samplers share the same short name + sampler_name_count[s_short] = sampler_name_count.get(s_short, 0) + 1 + if sampler_name_count[s_short] > 1: + s_name = f'sampler-{s_short}-{sampler_name_count[s_short]}' + else: + s_name = f'sampler-{s_short}' + + s_engine = sampler_cfg.get('engine', 'vllm') + s_max_len = sampler_cfg.get('max_model_len', 16000) + + # Compute tp / dp / total GPUs: + # tp = tensor parallelism (GPUs per vLLM process, for large models) + # dp = data parallelism (number of independent inference replicas) + # total GPUs = tp * dp + s_tp = sampler_cfg.get('tp', 1) + s_dp = sampler_cfg.get('dp', None) + s_gpus = sampler_cfg.get('gpus', None) + + if s_dp is not None and s_gpus is not None: + # Both specified: validate consistency + s_tp = s_gpus // s_dp if s_tp == 1 else s_tp + elif s_gpus is not None: + # Only total GPUs specified: derive dp + s_dp = max(1, s_gpus // s_tp) + elif s_dp is not None: + # Only dp specified: derive total + s_gpus = s_tp * s_dp + else: + # Nothing specified: default to 1 GPU (tp=1, dp=1) + s_dp = 1 + s_gpus = s_tp * s_dp + + s_total_gpus = s_tp * s_dp + + # Build device_mesh: include tp_size when tp>1 so that + # world_size = tp*dp and slice_dp dispatch computes correct + # rank_stride for DP data sharding. + mesh_config: dict = {'device_type': 'cuda', 'dp_size': s_dp} + if s_tp > 1: + mesh_config['tp_size'] = s_tp + + sampler_app: dict = { + 'name': + s_name, + 'route_prefix': + f'/api/v1/sampler/{s_model_id}', + 'import_path': + 'sampler', + 'args': { + 'model_id': f'ms://{s_model_id}', + 'nproc_per_node': s_total_gpus, + 'sampler_type': s_engine, + 'device_group': { + 'name': s_name, + 'ranks': s_total_gpus, + 'device_type': 'cuda', + 'gpus_per_worker': s_tp, + }, + 'device_mesh': mesh_config, + 'queue_config': { + 'rps_limit': 100, + 'tps_limit': 100000, + }, + }, + 'deployments': [{ + 'name': 'SamplerManagement', + 'autoscaling_config': { + 'min_replicas': 1, + 'max_replicas': 1, + 'target_ongoing_requests': 16, + }, + 'ray_actor_options': { + 'num_cpus': 0.1, + 'runtime_env': { + 'env_vars': { + 'TWINKLE_TRUST_REMOTE_CODE': '1' + }, + }, + }, + }], + } + + # Add engine-specific args + if s_engine == 'vllm': + engine_args = { + 'max_model_len': s_max_len, + 'gpu_memory_utilization': 0.85, + 'enable_lora': True, + 'logprobs_mode': 'processed_logprobs', + } + # Set tensor_parallel_size when tp > 1 + if s_tp > 1: + engine_args['tensor_parallel_size'] = s_tp + sampler_app['args']['engine_args'] = engine_args + + gpu_apps.append((s_total_gpus, sampler_app)) + + # 3. Sort GPU apps by GPU count DESCENDING, then append in order. + # Largest PG deploys first → claims the largest node → avoids deadlock. + gpu_apps.sort(key=lambda x: x[0], reverse=True) + for _, app_cfg in gpu_apps: + applications.append(app_cfg) + + # 4. Processor service + applications.append({ + 'name': + 'processor', + 'route_prefix': + '/api/v1/processor', + 'import_path': + 'processor', + 'args': { + 'ncpu_proc_per_node': 2, + 'device_group': { + 'name': 'processor', + 'ranks': 2, + 'device_type': 'CPU', + }, + 'device_mesh': { + 'device_type': 'CPU', + 'dp_size': 2, + }, + }, + 'deployments': [{ + 'name': 'ProcessorManagement', + 'autoscaling_config': { + 'min_replicas': 1, + 'max_replicas': 1, + 'target_ongoing_requests': 128, + }, + 'ray_actor_options': { + 'num_cpus': 0.1 + }, + }], + }) + + # === Assemble final config === + config = { + 'proxy_location': 'EveryNode', + 'http_options': { + 'host': '0.0.0.0', + 'port': port, + }, + 'applications': applications, + } + + # Write to ~/.cache/twinkle/server_config.yaml + config_dir = Path.home() / '.cache' / 'twinkle' + config_dir.mkdir(parents=True, exist_ok=True) + config_path = config_dir / 'server_config.yaml' + with open(config_path, 'w') as f: + yaml.dump(config, f, default_flow_style=False, allow_unicode=True) + + return str(config_path) + + async def _tool_shutdown_server(self) -> dict: + """Shut down Twinkle Server and Ray cluster. DESTROYS GPU model state.""" + import subprocess as _sp + + def _shutdown(): + results = {} + + # 1. Try `serve shutdown` to cleanly stop Ray Serve deployments + try: + r = _sp.run(['serve', 'shutdown', '-y'], capture_output=True, text=True, timeout=30) + results['serve_shutdown'] = 'ok' if r.returncode == 0 else r.stderr.strip() + except (FileNotFoundError, OSError) as e: + results['serve_shutdown'] = f'skipped: {e}' + + # 2. Kill any remaining twinkle.server processes + try: + _sp.run(['pkill', '-f', 'twinkle.server'], capture_output=True, timeout=5) + except (FileNotFoundError, OSError): + pass + + # 3. Stop Ray cluster + try: + r = _sp.run(['ray', 'stop', '--force'], capture_output=True, text=True, timeout=15) + results['ray_stop'] = 'ok' if r.returncode == 0 else r.stderr.strip() + except (FileNotFoundError, OSError) as e: + results['ray_stop'] = f'failed: {e}' + + results['status'] = 'shutdown_complete' + results['warning'] = 'All GPU model state has been released.' + return results + + return await asyncio.get_event_loop().run_in_executor(None, _shutdown) + + async def _tool_list_supported_models(self, base_url: str | None = None) -> dict: + """Query the Twinkle server for supported models.""" + url = base_url or self._server_url or os.environ.get('TWINKLE_SERVER_URL') or 'http://localhost:8000' + + def _query(): + # Use a lightweight HTTP GET instead of init_twinkle_client() which + # creates a session + heartbeat thread that would leak since we never + # call close(). + import urllib.error + import urllib.request + + endpoint = f'{url}/api/v1/twinkle/get_server_capabilities' + req = urllib.request.Request(endpoint, method='GET') + resp = urllib.request.urlopen(req, timeout=10) + data = json.loads(resp.read().decode()) + models = data.get('supported_models', []) + # Each model entry may be a dict with 'model_name' or a plain string + model_names = [] + for m in models: + if isinstance(m, dict): + model_names.append(m.get('model_name', '')) + else: + model_names.append(str(m)) + return { + 'base_url': url, + 'supported_models': model_names, + } + + try: + return await asyncio.get_event_loop().run_in_executor(None, _query) + except Exception as e: + return {'error': f'Failed to query {url}: {e}'} + + async def _tool_get_cluster_info(self) -> dict: + """Query cluster resources: try Ray first, fall back to nvidia-smi.""" + + def _query(): + # 1. Try connecting to an existing Ray cluster + ray_info = self._try_ray_cluster() + if ray_info is not None: + ray_info['ray_active'] = True + return ray_info + + # 2. Ray not available — fall back to nvidia-smi + nvidia_info = self._try_nvidia_smi() + nvidia_info['ray_active'] = False + nvidia_info['hint'] = ('Ray cluster is not running. To use distributed training, ' + 'start Ray first: `ray start --head --num-gpus=N` or use ' + 'the server mode run.sh script.') + return nvidia_info + + return await asyncio.get_event_loop().run_in_executor(None, _query) + + @staticmethod + def _try_ray_cluster() -> dict | None: + """Attempt to query an existing Ray cluster. Returns None if unavailable.""" + try: + import ray + except ImportError: + return None + + import logging as _logging + + try: + if not ray.is_initialized(): + ray.init( + address='auto', + ignore_reinit_error=True, + _timeout_s=5, + logging_level=_logging.ERROR, + configure_logging=False, + ) + + resources = ray.cluster_resources() + available = ray.available_resources() + nodes = ray.nodes() + gpu_total = resources.get('GPU', 0) + gpu_available = available.get('GPU', 0) + gpu_types = set() + for node in nodes: + for key in node.get('Resources', {}): + if key.startswith('accelerator_type:'): + gpu_types.add(key.split(':', 1)[1]) + return { + 'num_nodes': len([n for n in nodes if n.get('Alive')]), + 'gpu_total': int(gpu_total), + 'gpu_available': int(gpu_available), + 'gpu_types': sorted(gpu_types) if gpu_types else ['unknown'], + 'cpu_total': resources.get('CPU', 0), + 'memory_bytes': resources.get('memory', 0), + } + except Exception: + try: + import ray as _ray + if _ray.is_initialized(): + _ray.shutdown() + except Exception: + pass + return None + + @staticmethod + def _try_nvidia_smi() -> dict: + """Parse nvidia-smi output for local GPU info.""" + import subprocess as _sp + + try: + result = _sp.run( + [ + 'nvidia-smi', '--query-gpu=index,name,memory.total,memory.free,utilization.gpu', + '--format=csv,noheader,nounits' + ], + capture_output=True, + text=True, + timeout=10, + ) + if result.returncode != 0: + return {'error': f'nvidia-smi failed: {result.stderr.strip()}', 'gpu_total': 0} + + gpus = [] + for line in result.stdout.strip().split('\n'): + if not line.strip(): + continue + parts = [p.strip() for p in line.split(',')] + if len(parts) >= 5: + try: + gpus.append({ + 'index': int(parts[0]), + 'name': parts[1], + 'memory_total_mb': int(parts[2]), + 'memory_free_mb': int(parts[3]), + 'utilization_pct': int(parts[4]) if parts[4].isdigit() else 0, + }) + except (ValueError, IndexError): + # Skip lines with unparseable values (e.g. [N/A]) + continue + + gpu_types = sorted({g['name'] for g in gpus}) + return { + 'gpu_total': len(gpus), + 'gpu_available': len([g for g in gpus if g['utilization_pct'] < 10]), + 'gpu_types': gpu_types if gpu_types else ['none'], + 'gpus': gpus, + 'source': 'nvidia-smi', + } + except FileNotFoundError: + return {'error': 'nvidia-smi not found (no NVIDIA GPU or driver not installed)', 'gpu_total': 0} + except Exception as e: + return {'error': f'nvidia-smi query failed: {e}', 'gpu_total': 0} diff --git a/src/twinkle_client/auto/agent/tool_schemas.py b/src/twinkle_client/auto/agent/tool_schemas.py new file mode 100644 index 00000000..7db691fa --- /dev/null +++ b/src/twinkle_client/auto/agent/tool_schemas.py @@ -0,0 +1,338 @@ +# Copyright (c) Twinkle Contributors. All rights reserved. +"""OpenAI function-calling schemas exposed by the auto agent.""" +from __future__ import annotations + +from typing import Any + +TOOL_SCHEMAS: list[dict[str, Any]] = [ + { + 'type': 'function', + 'function': { + 'name': 'list_training_runs', + 'description': 'List all active and historical training runs.', + 'parameters': { + 'type': 'object', + 'properties': {}, + 'required': [] + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': 'get_training_status', + 'description': 'Get detailed status and recent metrics for a training run.', + 'parameters': { + 'type': 'object', + 'properties': { + 'run_id': { + 'type': 'string', + 'description': 'Training run ID.' + }, + }, + 'required': ['run_id'], + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': + 'start_server', + 'description': ('Start Ray cluster and Twinkle Server. MUST be called before start_training. ' + 'Idempotent: skips if server is already reachable. ' + 'Supports multi-model deployments: one training model + N sampler/teacher models. ' + 'Automatically generates server_config.yaml from parameters.'), + 'parameters': { + 'type': 'object', + 'properties': { + 'model_id': { + 'type': 'string', + 'description': 'Student/training model ID (e.g. "Qwen/Qwen3.5-4B").', + }, + 'train_gpus': { + 'type': 'integer', + 'description': 'GPUs for the training model. Default: auto-detect remaining GPUs.', + }, + 'backend': { + 'type': 'string', + 'enum': ['transformers', 'megatron'], + 'description': 'Training model backend. Default: transformers.', + }, + 'samplers': { + 'type': + 'array', + 'description': ('List of sampler/teacher models for RL/OPD. Each entry deploys ' + 'an inference service (vLLM or torch). Omit for simple SFT.'), + 'items': { + 'type': 'object', + 'properties': { + 'model_id': { + 'type': 'string', + 'description': 'Teacher/reference model ID (e.g. "Qwen/Qwen3.5-72B").', + }, + 'gpus': { + 'type': 'integer', + 'description': + 'Total number of GPUs for this sampler. Default: 1. Must equal tp * dp.', + }, + 'tp': { + 'type': + 'integer', + 'description': + ('Tensor parallelism size (GPUs per vLLM worker process). ' + 'Use tp>1 for large models that do not fit on a single GPU. Default: 1.'), + }, + 'dp': { + 'type': + 'integer', + 'description': ('Data parallelism size (number of independent inference replicas). ' + 'If not specified, computed as gpus // tp. Default: 1.'), + }, + 'engine': { + 'type': 'string', + 'enum': ['vllm', 'torch'], + 'description': 'Inference engine. Default: vllm.', + }, + 'max_model_len': { + 'type': 'integer', + 'description': 'Max sequence length for inference. Default: 16000.', + }, + }, + 'required': ['model_id'], + }, + }, + 'port': { + 'type': 'integer', + 'description': 'HTTP port for server. Default: 8000.', + }, + }, + 'required': ['model_id'], + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': + 'shutdown_server', + 'description': ('Shut down Twinkle Server and Ray cluster. WARNING: This releases all GPU resources ' + 'and DESTROYS model state held in server memory. Only call when training is truly ' + 'finished and you no longer need the server. Model weights/optimizer state in GPU ' + 'will be LOST unless a checkpoint was explicitly saved.'), + 'parameters': { + 'type': 'object', + 'properties': {}, + 'required': [] + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': + 'start_training', + 'description': ('Create a new training run: write the client script, launch it, and start monitoring. ' + 'REQUIRES: Twinkle Server must be running (call start_server first). ' + 'The client script connects to the server — server holds model state in GPU memory. ' + 'Kill client = pause (state preserved). Re-launch client = resume.'), + 'parameters': { + 'type': 'object', + 'properties': { + 'run_id': { + 'type': 'string', + 'description': 'Unique run ID (e.g., "grpo-gsm8k").' + }, + 'script_content': { + 'type': 'string', + 'description': 'Full Python source code of the training script.' + }, + 'model_id': { + 'type': 'string', + 'description': 'Model identifier for metadata (e.g., "Qwen/Qwen3.5-4B").' + }, + }, + 'required': ['run_id', 'script_content'], + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': 'select_run', + 'description': 'Switch to monitor a different training run. Updates connection context.', + 'parameters': { + 'type': 'object', + 'properties': { + 'run_id': { + 'type': 'string', + 'description': 'Training run ID to monitor.' + }, + }, + 'required': ['run_id'], + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': + 'pause_training', + 'description': ('Pause training by killing the client process (SIGKILL). ' + 'Server retains all state — call resume_training to continue.'), + 'parameters': { + 'type': 'object', + 'properties': { + 'run_id': { + 'type': 'string', + 'description': 'Training run ID to pause.' + }, + }, + 'required': ['run_id'], + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': 'resume_training', + 'description': 'Resume a paused training run by re-launching the client script. Server state is preserved.', + 'parameters': { + 'type': 'object', + 'properties': { + 'run_id': { + 'type': 'string', + 'description': 'Training run ID to resume.' + }, + }, + 'required': ['run_id'], + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': + 'stop_training', + 'description': ('Gracefully stop the training client (SIGTERM). The script saves a checkpoint ' + 'before exiting. Server retains model/optimizer state in GPU memory — ' + 'use resume_training to continue. Similar to pause_training but with checkpoint save. ' + 'To fully release GPU resources, use shutdown_server.'), + 'parameters': { + 'type': 'object', + 'properties': { + 'run_id': { + 'type': 'string', + 'description': 'Training run ID to stop.' + }, + }, + 'required': ['run_id'], + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': + 'update_script', + 'description': + ('Update the training script for a run. Archives the current train.py as train_v{N}.py ' + 'and writes the new version. Use after diagnosing a script error, then call resume_training.'), + 'parameters': { + 'type': 'object', + 'properties': { + 'run_id': { + 'type': 'string', + 'description': 'Training run ID.' + }, + 'script_content': { + 'type': 'string', + 'description': 'Full Python source code of the new training script.' + }, + }, + 'required': ['run_id', 'script_content'], + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': + 'list_supported_models', + 'description': ('Query the Twinkle server for its list of supported base models. ' + 'Always call this before writing a training script to verify model availability.'), + 'parameters': { + 'type': 'object', + 'properties': { + 'base_url': { + 'type': + 'string', + 'description': + 'Server base URL. Default: http://localhost:8000. Cloud: http://www.modelscope.cn/twinkle', + }, + }, + 'required': [], + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': 'search_datasets', + 'description': 'Search ModelScope Hub for datasets matching a query.', + 'parameters': { + 'type': 'object', + 'properties': { + 'query': { + 'type': 'string', + 'description': 'Search query for datasets.' + }, + 'limit': { + 'type': 'integer', + 'description': 'Max results (default 5).' + }, + }, + 'required': ['query'], + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': 'search_models', + 'description': 'Search ModelScope Hub for models matching a query.', + 'parameters': { + 'type': 'object', + 'properties': { + 'query': { + 'type': 'string', + 'description': 'Search query for models.' + }, + 'limit': { + 'type': 'integer', + 'description': 'Max results (default 5).' + }, + }, + 'required': ['query'], + }, + }, + }, + { + 'type': 'function', + 'function': { + 'name': + 'get_cluster_info', + 'description': ('Get cluster GPU resource info for planning training parallelism. ' + 'First attempts to query a running Ray cluster; if Ray is not available, ' + 'falls back to nvidia-smi for local GPU discovery. ' + 'The result indicates whether Ray is active — if not, the training script ' + 'should either start a local Ray cluster itself or the user should launch ' + 'Ray manually (see server mode run.sh).'), + 'parameters': { + 'type': 'object', + 'properties': {}, + 'required': [] + }, + }, + }, +] diff --git a/src/twinkle_client/auto/agent/tools.py b/src/twinkle_client/auto/agent/tools.py index 9a3c7388..40b08f7c 100644 --- a/src/twinkle_client/auto/agent/tools.py +++ b/src/twinkle_client/auto/agent/tools.py @@ -3,355 +3,20 @@ from __future__ import annotations -import asyncio import json import os from typing import Any, Callable from twinkle.utils.logger import get_logger from twinkle_client.auto.connection import LocalConnection +from .search_tools import _SearchTools +from .server_tools import _ServerTools +from .tool_schemas import TOOL_SCHEMAS logger = get_logger() -# ────────────────────────────────────────────────────────────────────────────── -# Tool schemas (OpenAI function calling format) -# ────────────────────────────────────────────────────────────────────────────── -TOOL_SCHEMAS: list[dict[str, Any]] = [ - { - 'type': 'function', - 'function': { - 'name': 'list_training_runs', - 'description': 'List all active and historical training runs.', - 'parameters': { - 'type': 'object', - 'properties': {}, - 'required': [] - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': 'get_training_status', - 'description': 'Get detailed status and recent metrics for a training run.', - 'parameters': { - 'type': 'object', - 'properties': { - 'run_id': { - 'type': 'string', - 'description': 'Training run ID.' - }, - }, - 'required': ['run_id'], - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': - 'start_server', - 'description': ('Start Ray cluster and Twinkle Server. MUST be called before start_training. ' - 'Idempotent: skips if server is already reachable. ' - 'Supports multi-model deployments: one training model + N sampler/teacher models. ' - 'Automatically generates server_config.yaml from parameters.'), - 'parameters': { - 'type': 'object', - 'properties': { - 'model_id': { - 'type': 'string', - 'description': 'Student/training model ID (e.g. "Qwen/Qwen3.5-4B").', - }, - 'train_gpus': { - 'type': 'integer', - 'description': 'GPUs for the training model. Default: auto-detect remaining GPUs.', - }, - 'backend': { - 'type': 'string', - 'enum': ['transformers', 'megatron'], - 'description': 'Training model backend. Default: transformers.', - }, - 'samplers': { - 'type': - 'array', - 'description': ('List of sampler/teacher models for RL/OPD. Each entry deploys ' - 'an inference service (vLLM or torch). Omit for simple SFT.'), - 'items': { - 'type': 'object', - 'properties': { - 'model_id': { - 'type': 'string', - 'description': 'Teacher/reference model ID (e.g. "Qwen/Qwen3.5-72B").', - }, - 'gpus': { - 'type': 'integer', - 'description': - 'Total number of GPUs for this sampler. Default: 1. Must equal tp * dp.', - }, - 'tp': { - 'type': - 'integer', - 'description': - ('Tensor parallelism size (GPUs per vLLM worker process). ' - 'Use tp>1 for large models that do not fit on a single GPU. Default: 1.'), - }, - 'dp': { - 'type': - 'integer', - 'description': ('Data parallelism size (number of independent inference replicas). ' - 'If not specified, computed as gpus // tp. Default: 1.'), - }, - 'engine': { - 'type': 'string', - 'enum': ['vllm', 'torch'], - 'description': 'Inference engine. Default: vllm.', - }, - 'max_model_len': { - 'type': 'integer', - 'description': 'Max sequence length for inference. Default: 16000.', - }, - }, - 'required': ['model_id'], - }, - }, - 'port': { - 'type': 'integer', - 'description': 'HTTP port for server. Default: 8000.', - }, - }, - 'required': ['model_id'], - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': - 'shutdown_server', - 'description': ('Shut down Twinkle Server and Ray cluster. WARNING: This releases all GPU resources ' - 'and DESTROYS model state held in server memory. Only call when training is truly ' - 'finished and you no longer need the server. Model weights/optimizer state in GPU ' - 'will be LOST unless a checkpoint was explicitly saved.'), - 'parameters': { - 'type': 'object', - 'properties': {}, - 'required': [] - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': - 'start_training', - 'description': ('Create a new training run: write the client script, launch it, and start monitoring. ' - 'REQUIRES: Twinkle Server must be running (call start_server first). ' - 'The client script connects to the server — server holds model state in GPU memory. ' - 'Kill client = pause (state preserved). Re-launch client = resume.'), - 'parameters': { - 'type': 'object', - 'properties': { - 'run_id': { - 'type': 'string', - 'description': 'Unique run ID (e.g., "grpo-gsm8k").' - }, - 'script_content': { - 'type': 'string', - 'description': 'Full Python source code of the training script.' - }, - 'model_id': { - 'type': 'string', - 'description': 'Model identifier for metadata (e.g., "Qwen/Qwen3.5-4B").' - }, - }, - 'required': ['run_id', 'script_content'], - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': 'select_run', - 'description': 'Switch to monitor a different training run. Updates connection context.', - 'parameters': { - 'type': 'object', - 'properties': { - 'run_id': { - 'type': 'string', - 'description': 'Training run ID to monitor.' - }, - }, - 'required': ['run_id'], - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': 'pause_training', - 'description': 'Pause training by killing the client process (SIGKILL). Server retains all state — call ' - 'resume_training to continue.', - 'parameters': { - 'type': 'object', - 'properties': { - 'run_id': { - 'type': 'string', - 'description': 'Training run ID to pause.' - }, - }, - 'required': ['run_id'], - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': 'resume_training', - 'description': 'Resume a paused training run by re-launching the client script. Server state is preserved.', - 'parameters': { - 'type': 'object', - 'properties': { - 'run_id': { - 'type': 'string', - 'description': 'Training run ID to resume.' - }, - }, - 'required': ['run_id'], - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': - 'stop_training', - 'description': ('Gracefully stop the training client (SIGTERM). The script saves a checkpoint ' - 'before exiting. Server retains model/optimizer state in GPU memory — ' - 'use resume_training to continue. Similar to pause_training but with checkpoint save. ' - 'To fully release GPU resources, use shutdown_server.'), - 'parameters': { - 'type': 'object', - 'properties': { - 'run_id': { - 'type': 'string', - 'description': 'Training run ID to stop.' - }, - }, - 'required': ['run_id'], - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': 'update_script', - 'description': 'Update the training script for a run. Archives the current train.py as train_v{N}.py and ' - 'writes the new version. Use after diagnosing a script error, then call resume_training.', - 'parameters': { - 'type': 'object', - 'properties': { - 'run_id': { - 'type': 'string', - 'description': 'Training run ID.' - }, - 'script_content': { - 'type': 'string', - 'description': 'Full Python source code of the new training script.' - }, - }, - 'required': ['run_id', 'script_content'], - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': 'list_supported_models', - 'description': 'Query the Twinkle server for its list of supported base models. Always call this before ' - 'writing a training script to verify model availability.', - 'parameters': { - 'type': 'object', - 'properties': { - 'base_url': { - 'type': - 'string', - 'description': - 'Server base URL. Default: http://localhost:8000. Cloud: http://www.modelscope.cn/twinkle', - }, - }, - 'required': [], - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': 'search_datasets', - 'description': 'Search ModelScope Hub for datasets matching a query.', - 'parameters': { - 'type': 'object', - 'properties': { - 'query': { - 'type': 'string', - 'description': 'Search query for datasets.' - }, - 'limit': { - 'type': 'integer', - 'description': 'Max results (default 5).' - }, - }, - 'required': ['query'], - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': 'search_models', - 'description': 'Search ModelScope Hub for models matching a query.', - 'parameters': { - 'type': 'object', - 'properties': { - 'query': { - 'type': 'string', - 'description': 'Search query for models.' - }, - 'limit': { - 'type': 'integer', - 'description': 'Max results (default 5).' - }, - }, - 'required': ['query'], - }, - }, - }, - { - 'type': 'function', - 'function': { - 'name': - 'get_cluster_info', - 'description': ('Get cluster GPU resource info for planning training parallelism. ' - 'First attempts to query a running Ray cluster; if Ray is not available, ' - 'falls back to nvidia-smi for local GPU discovery. ' - 'The result indicates whether Ray is active — if not, the training script ' - 'should either start a local Ray cluster itself or the user should launch ' - 'Ray manually (see server mode run.sh).'), - 'parameters': { - 'type': 'object', - 'properties': {}, - 'required': [] - }, - }, - }, -] - -# ────────────────────────────────────────────────────────────────────────────── -# Tool executor -# ────────────────────────────────────────────────────────────────────────────── - - -class ToolExecutor: +class ToolExecutor(_ServerTools, _SearchTools): """Executes agent tool calls against the local connection.""" def __init__(self, connection: LocalConnection): @@ -424,854 +89,3 @@ async def _tool_stop_training(self, run_id: str) -> dict: async def _tool_update_script(self, run_id: str, script_content: str) -> dict: return self.connection.update_script(run_id, script_content) - - # ── Server lifecycle ── - - async def _check_server_health(self, url: str) -> bool: - """Check if Twinkle Server is reachable (non-blocking).""" - import urllib.error - import urllib.request - - def _probe(): - try: - req = urllib.request.Request(f'{url}/api/v1/healthz', method='GET') - urllib.request.urlopen(req, timeout=3) - return True - except (urllib.error.URLError, OSError): - # Try a simpler connectivity check - try: - urllib.request.urlopen(url, timeout=3) - return True - except (urllib.error.URLError, OSError): - return False - - return await asyncio.get_event_loop().run_in_executor(None, _probe) - - # ── Server startup pipeline ── - - async def _tool_start_server( - self, - model_id: str, - train_gpus: int | None = None, - port: int = 8000, - backend: str = 'transformers', - samplers: list[dict] | None = None, - ) -> dict: - """Start Ray cluster + Twinkle Server. Idempotent. Supports multi-model.""" - server_url = self._server_url or os.environ.get('TWINKLE_SERVER_URL') or f'http://localhost:{port}' - - # Idempotent: skip if already running - if await self._check_server_health(server_url): - self._server_url = server_url - return {'status': 'already_running', 'server_url': server_url} - - def _start(): - sampler_list = samplers or [] - - # Step 1: Detect hardware & compute GPU partition - total_hw_gpus = self._detect_gpu_count() - if total_hw_gpus == 0: - return {'status': 'error', 'error': 'No GPUs detected. Cannot start training server.'} - - alloc = self._compute_gpu_allocation(sampler_list, train_gpus, total_hw_gpus) - if 'error' in alloc: - return {'status': 'error', 'error': alloc['error']} - t_gpus, sampler_gpu_total = alloc['train_gpus'], alloc['sampler_gpus'] - - # Step 2: Generate server_config.yaml - config_path = self._generate_server_config( - model_id=model_id, - train_gpus=t_gpus, - port=port, - backend=backend, - samplers=sampler_list, - ) - - # Step 3: Start Ray cluster (multi-node GPU partitioning) - ray_err = self._start_ray_cluster(t_gpus, sampler_gpu_total) - if ray_err: - return {'status': 'error', 'error': ray_err} - - # Step 4: Launch Twinkle Server process - proc, log_path, err = self._launch_server_process(config_path) - if err: - return {'status': 'error', 'error': err} - - # Step 5: Wait for readiness (healthz + sampler engine) - return self._wait_server_ready( - server_url=server_url, - proc=proc, - log_path=log_path, - sampler_list=sampler_list, - model_id=model_id, - t_gpus=t_gpus, - backend=backend, - config_path=config_path, - ) - - result = await asyncio.get_event_loop().run_in_executor(None, _start) - if result.get('status') in ('started', 'already_running'): - self._server_url = server_url - return result - - # ── Server startup helpers ── - - @staticmethod - def _detect_gpu_count() -> int: - """Detect total hardware GPU count via nvidia-smi.""" - import subprocess as _sp - try: - r = _sp.run( - ['nvidia-smi', '--query-gpu=index', '--format=csv,noheader'], - capture_output=True, - text=True, - timeout=10, - ) - if r.returncode == 0: - return len([ln for ln in r.stdout.strip().split('\n') if ln.strip()]) - except (FileNotFoundError, OSError): - pass - return 0 - - @staticmethod - def _compute_gpu_allocation( - sampler_list: list[dict], - train_gpus: int | None, - total_hw_gpus: int, - ) -> dict: - """Compute GPU partition: {train_gpus, sampler_gpus} or {error}.""" - sampler_gpu_total = 0 - for s in sampler_list: - s_tp = s.get('tp', 1) - s_dp, s_gpus = s.get('dp'), s.get('gpus') - if s_gpus is not None: - sampler_gpu_total += s_gpus - elif s_dp is not None: - sampler_gpu_total += s_tp * s_dp - else: - sampler_gpu_total += s_tp # default dp=1 - - t_gpus = train_gpus if train_gpus is not None else max(1, total_hw_gpus - sampler_gpu_total) - needed = t_gpus + sampler_gpu_total - if needed > total_hw_gpus: - return { - 'error': (f'Requested {needed} GPUs (train={t_gpus}, samplers={sampler_gpu_total}) ' - f'but only {total_hw_gpus} available.'), - } - return {'train_gpus': t_gpus, 'sampler_gpus': sampler_gpu_total} - - @staticmethod - def _start_ray_cluster(train_gpus: int, sampler_gpus: int) -> str | None: - """Start Ray multi-node cluster with GPU partitioning. - - Each role gets its own Ray node with dedicated CUDA_VISIBLE_DEVICES - so GPUs are indexed from 0 within each node. This prevents the - GPU ID mapping issues that occur with a single-node setup. - - On a single machine, multiple raylets need separate --temp-dir to - avoid being detected as "already running". - - Returns an error message on failure, or None on success. - """ - import subprocess as _sp - import tempfile - from pathlib import Path - - _sp.run(['ray', 'stop', '--force'], capture_output=True, timeout=15) - - # Create unique temp dirs so each `ray start` spawns a separate raylet - ray_base = Path(tempfile.gettempdir()) / 'twinkle_ray' - ray_base.mkdir(parents=True, exist_ok=True) - - def _ray_node( - devices: str, - num_gpus: int, - *, - head: bool = False, - node_name: str = 'worker', - ) -> str | None: - env = os.environ.copy() - env['CUDA_VISIBLE_DEVICES'] = devices - temp_dir = str(ray_base / node_name) - cmd = ['ray', 'start', f'--temp-dir={temp_dir}'] - if head: - cmd += ['--head', '--port=6379', '--disable-usage-stats', '--include-dashboard=false'] - else: - cmd += ['--address=127.0.0.1:6379'] - cmd.append(f'--num-gpus={num_gpus}') - r = _sp.run(cmd, capture_output=True, text=True, timeout=30, env=env) - if r.returncode != 0 and 'already' not in r.stderr.lower(): - return r.stderr.strip() - return None - - # Head node — training model GPUs - model_devices = ','.join(str(i) for i in range(train_gpus)) - err = _ray_node(model_devices, train_gpus, head=True, node_name='head') - if err: - return f'Ray head start failed: {err}' - - # GPU Worker node — sampler GPUs - if sampler_gpus > 0: - sampler_devices = ','.join(str(i) for i in range(train_gpus, train_gpus + sampler_gpus)) - err = _ray_node(sampler_devices, sampler_gpus, node_name='gpu_worker') - if err: - return f'Ray GPU worker start failed: {err}' - - # CPU Worker node — processor (no GPU) - _ray_node('', 0, node_name='cpu_worker') - return None - - @staticmethod - def _launch_server_process(config_path: str) -> tuple: - """Launch Twinkle Server as a detached background process. - - Returns (proc, log_path, error). On success error is None. - """ - import subprocess as _sp - from pathlib import Path - - log_dir = Path.home() / '.cache' / 'twinkle' - log_dir.mkdir(parents=True, exist_ok=True) - log_path = str(log_dir / 'server.log') - log_file = open(log_path, 'w') - - cmd = ['python', '-m', 'twinkle.server', 'launch', '--config', config_path] - try: - proc = _sp.Popen( - cmd, - stdout=log_file, - stderr=_sp.STDOUT, - start_new_session=True, - ) - except OSError as e: - log_file.close() - return None, log_path, f'Failed to start Twinkle server: {e}' - return proc, log_path, None - - @staticmethod - def _wait_server_ready( - server_url: str, - proc, - log_path: str, - sampler_list: list[dict], - model_id: str, - t_gpus: int, - backend: str, - config_path: str, - ) -> dict: - """Poll server until healthy (healthz + sampler engine ready).""" - import time - import urllib.error - import urllib.request - - timeout_s = 120 if sampler_list else 60 - needed = t_gpus + sum(s.get('gpus') or (s.get('tp', 1) * s.get('dp', 1)) for s in sampler_list) - - for _ in range(timeout_s): - time.sleep(1) - if proc.poll() is not None: - # Server died — read log tail to diagnose - log_tail = ToolExecutor._read_log_tail(log_path, max_chars=2000) - error_msg = (f'Server exited immediately (code={proc.returncode}). ' - f'Model: {model_id}, GPUs: {t_gpus}, Samplers: {len(sampler_list)}.\n' - f'--- server.log tail ---\n{log_tail}') - return { - 'status': 'error', - 'error': error_msg, - 'log_path': log_path, - 'hint': 'Check if required packages are installed (pip install -e ".[all]").', - } - try: - urllib.request.urlopen(f'{server_url}/api/v1/healthz', timeout=2) - except (OSError, Exception): - continue - - # healthz OK — additionally wait for sampler vLLM engines - if sampler_list and not ToolExecutor._probe_sampler_ready(server_url, sampler_list, model_id): - return { - 'status': 'started', - 'warning': 'Server is up but sampler may still be loading.', - 'server_url': server_url, - 'server_pid': proc.pid, - 'model_id': model_id, - 'log_path': log_path, - } - - return { - 'status': 'started', - 'server_url': server_url, - 'server_pid': proc.pid, - 'model_id': model_id, - 'train_gpus': t_gpus, - 'backend': backend, - 'samplers': [s.get('model_id') for s in sampler_list], - 'total_gpus_used': needed, - 'config_path': config_path, - 'log_path': log_path, - } - - return { - 'status': 'timeout', - 'error': 'Health check did not pass within timeout. Models may still be loading.', - 'server_pid': proc.pid, - 'log_path': log_path, - } - - @staticmethod - def _read_log_tail(log_path: str, max_chars: int = 2000) -> str: - """Read the tail of a log file for error diagnosis.""" - try: - with open(log_path, errors='replace') as f: - content = f.read() - if len(content) <= max_chars: - return content.strip() - return content[-max_chars:].strip() - except OSError: - return '(could not read log file)' - - @staticmethod - def _probe_sampler_ready(server_url: str, sampler_list: list[dict], fallback_model_id: str) -> bool: - """Probe sampler route up to 90s to confirm vLLM engine is loaded.""" - import time - import urllib.error - import urllib.request - - s_mid = sampler_list[0].get('model_id', fallback_model_id) - probe_url = f'{server_url}/api/v1/sampler/{s_mid}/twinkle/create' - - for _ in range(90): - try: - req = urllib.request.Request( - probe_url, - method='POST', - data=b'{}', - headers={'Content-Type': 'application/json'}, - ) - urllib.request.urlopen(req, timeout=5) - return True # non-error response = ready - except urllib.error.HTTPError as e: - if e.code < 500: - return True # 4xx = actor alive, just bad request - time.sleep(1) # 5xx = still loading - except (OSError, Exception): - time.sleep(1) - return False - - @staticmethod - def _generate_server_config( - model_id: str, - train_gpus: int, - port: int = 8000, - backend: str = 'transformers', - samplers: list[dict] | None = None, - ) -> str: - """Generate a server_config.yaml from template and return its path. - - Supports multi-model topology: - - 1 training model (student) - - N sampler/teacher models (for RL/OPD) - - 1 processor service - """ - import yaml - from pathlib import Path - - sampler_list = samplers or [] - - # Sanitize model name for use in route/names - def _short(mid: str) -> str: - return mid.split('/')[-1] if '/' in mid else mid - - model_short = _short(model_id) - - # Collect all model IDs for supported_models - all_model_ids = [model_id] + [s['model_id'] for s in sampler_list] - - # === Build applications list === - applications = [] - - # 1. API Gateway - applications.append({ - 'name': - 'server', - 'route_prefix': - '/api/v1', - 'import_path': - 'server', - 'args': { - 'server_config': { - 'per_token_model_limit': 3 - }, - 'supported_models': all_model_ids, - }, - 'deployments': [{ - 'name': 'TinkerCompatServer', - 'max_ongoing_requests': 50, - 'autoscaling_config': { - 'min_replicas': 1, - 'max_replicas': 1, - 'target_ongoing_requests': 128, - }, - 'ray_actor_options': { - 'num_cpus': 0.1 - }, - }], - }) - - # 2. Build GPU-requiring applications (model + samplers), - # then sort by GPU count DESCENDING before appending. - # Largest PG deploys first → it has the fewest node choices → - # avoids GPU scheduling deadlock on single-machine multi-node. - gpu_apps: list[tuple[int, dict]] = [] # (gpu_count, app_config) - - # 2a. Training model worker (student) - gpu_apps.append(( - train_gpus, - { - 'name': - f'models-{model_short}', - 'route_prefix': - f'/api/v1/model/{model_id}', - 'import_path': - 'model', - 'args': { - 'backend': backend, - 'model_id': f'ms://{model_id}', - 'max_length': 500000, # total tokens per forward pass (must match max_input_tokens) - 'nproc_per_node': train_gpus, - 'device_group': { - 'name': 'model', - 'ranks': train_gpus, - 'device_type': 'cuda', - }, - 'device_mesh': { - 'device_type': 'cuda', - 'dp_size': train_gpus, - }, - 'queue_config': { - 'rps_limit': 100, - 'tps_limit': 100000, - 'max_input_tokens': 500000, - }, - 'adapter_config': { - 'adapter_timeout': 600, - }, - }, - 'deployments': [{ - 'name': 'ModelManagement', - 'autoscaling_config': { - 'min_replicas': 1, - 'max_replicas': 1, - 'target_ongoing_requests': 16, - }, - 'ray_actor_options': { - 'num_cpus': 0.1, - 'runtime_env': { - 'env_vars': { - 'TWINKLE_TRUST_REMOTE_CODE': '1' - }, - }, - }, - }], - })) - - # 2b. Sampler/teacher models - sampler_name_count: dict[str, int] = {} - for sampler_cfg in sampler_list: - s_model_id = sampler_cfg['model_id'] - s_short = _short(s_model_id) - - # Deduplicate names when multiple samplers share the same short name - sampler_name_count[s_short] = sampler_name_count.get(s_short, 0) + 1 - if sampler_name_count[s_short] > 1: - s_name = f'sampler-{s_short}-{sampler_name_count[s_short]}' - else: - s_name = f'sampler-{s_short}' - - s_engine = sampler_cfg.get('engine', 'vllm') - s_max_len = sampler_cfg.get('max_model_len', 16000) - - # Compute tp / dp / total GPUs: - # tp = tensor parallelism (GPUs per vLLM process, for large models) - # dp = data parallelism (number of independent inference replicas) - # total GPUs = tp * dp - s_tp = sampler_cfg.get('tp', 1) - s_dp = sampler_cfg.get('dp', None) - s_gpus = sampler_cfg.get('gpus', None) - - if s_dp is not None and s_gpus is not None: - # Both specified: validate consistency - s_tp = s_gpus // s_dp if s_tp == 1 else s_tp - elif s_gpus is not None: - # Only total GPUs specified: derive dp - s_dp = max(1, s_gpus // s_tp) - elif s_dp is not None: - # Only dp specified: derive total - s_gpus = s_tp * s_dp - else: - # Nothing specified: default to 1 GPU (tp=1, dp=1) - s_dp = 1 - s_gpus = s_tp * s_dp - - s_total_gpus = s_tp * s_dp - - # Build device_mesh: include tp_size when tp>1 so that - # world_size = tp*dp and slice_dp dispatch computes correct - # rank_stride for DP data sharding. - mesh_config: dict = {'device_type': 'cuda', 'dp_size': s_dp} - if s_tp > 1: - mesh_config['tp_size'] = s_tp - - sampler_app: dict = { - 'name': - s_name, - 'route_prefix': - f'/api/v1/sampler/{s_model_id}', - 'import_path': - 'sampler', - 'args': { - 'model_id': f'ms://{s_model_id}', - 'nproc_per_node': s_total_gpus, - 'sampler_type': s_engine, - 'device_group': { - 'name': s_name, - 'ranks': s_total_gpus, - 'device_type': 'cuda', - 'gpus_per_worker': s_tp, - }, - 'device_mesh': mesh_config, - 'queue_config': { - 'rps_limit': 100, - 'tps_limit': 100000, - }, - }, - 'deployments': [{ - 'name': 'SamplerManagement', - 'autoscaling_config': { - 'min_replicas': 1, - 'max_replicas': 1, - 'target_ongoing_requests': 16, - }, - 'ray_actor_options': { - 'num_cpus': 0.1, - 'runtime_env': { - 'env_vars': { - 'TWINKLE_TRUST_REMOTE_CODE': '1' - }, - }, - }, - }], - } - - # Add engine-specific args - if s_engine == 'vllm': - engine_args = { - 'max_model_len': s_max_len, - 'gpu_memory_utilization': 0.85, - 'enable_lora': True, - 'logprobs_mode': 'processed_logprobs', - } - # Set tensor_parallel_size when tp > 1 - if s_tp > 1: - engine_args['tensor_parallel_size'] = s_tp - sampler_app['args']['engine_args'] = engine_args - - gpu_apps.append((s_total_gpus, sampler_app)) - - # 3. Sort GPU apps by GPU count DESCENDING, then append in order. - # Largest PG deploys first → claims the largest node → avoids deadlock. - gpu_apps.sort(key=lambda x: x[0], reverse=True) - for _, app_cfg in gpu_apps: - applications.append(app_cfg) - - # 4. Processor service - applications.append({ - 'name': - 'processor', - 'route_prefix': - '/api/v1/processor', - 'import_path': - 'processor', - 'args': { - 'ncpu_proc_per_node': 2, - 'device_group': { - 'name': 'processor', - 'ranks': 2, - 'device_type': 'CPU', - }, - 'device_mesh': { - 'device_type': 'CPU', - 'dp_size': 2, - }, - }, - 'deployments': [{ - 'name': 'ProcessorManagement', - 'autoscaling_config': { - 'min_replicas': 1, - 'max_replicas': 1, - 'target_ongoing_requests': 128, - }, - 'ray_actor_options': { - 'num_cpus': 0.1 - }, - }], - }) - - # === Assemble final config === - config = { - 'proxy_location': 'EveryNode', - 'http_options': { - 'host': '0.0.0.0', - 'port': port, - }, - 'applications': applications, - } - - # Write to ~/.cache/twinkle/server_config.yaml - config_dir = Path.home() / '.cache' / 'twinkle' - config_dir.mkdir(parents=True, exist_ok=True) - config_path = config_dir / 'server_config.yaml' - with open(config_path, 'w') as f: - yaml.dump(config, f, default_flow_style=False, allow_unicode=True) - - return str(config_path) - - async def _tool_shutdown_server(self) -> dict: - """Shut down Twinkle Server and Ray cluster. DESTROYS GPU model state.""" - import subprocess as _sp - - def _shutdown(): - results = {} - - # 1. Try `serve shutdown` to cleanly stop Ray Serve deployments - try: - r = _sp.run(['serve', 'shutdown', '-y'], capture_output=True, text=True, timeout=30) - results['serve_shutdown'] = 'ok' if r.returncode == 0 else r.stderr.strip() - except (FileNotFoundError, OSError) as e: - results['serve_shutdown'] = f'skipped: {e}' - - # 2. Kill any remaining twinkle.server processes - try: - _sp.run(['pkill', '-f', 'twinkle.server'], capture_output=True, timeout=5) - except (FileNotFoundError, OSError): - pass - - # 3. Stop Ray cluster - try: - r = _sp.run(['ray', 'stop', '--force'], capture_output=True, text=True, timeout=15) - results['ray_stop'] = 'ok' if r.returncode == 0 else r.stderr.strip() - except (FileNotFoundError, OSError) as e: - results['ray_stop'] = f'failed: {e}' - - results['status'] = 'shutdown_complete' - results['warning'] = 'All GPU model state has been released.' - return results - - return await asyncio.get_event_loop().run_in_executor(None, _shutdown) - - # ── Server queries ── - - async def _tool_list_supported_models(self, base_url: str | None = None) -> dict: - """Query the Twinkle server for supported models.""" - url = base_url or self._resolve_server_url() - - def _query(): - # Use a lightweight HTTP GET instead of init_twinkle_client() which - # creates a session + heartbeat thread that would leak since we never - # call close(). - import urllib.error - import urllib.request - - endpoint = f'{url}/api/v1/twinkle/get_server_capabilities' - req = urllib.request.Request(endpoint, method='GET') - resp = urllib.request.urlopen(req, timeout=10) - data = json.loads(resp.read().decode()) - models = data.get('supported_models', []) - # Each model entry may be a dict with 'model_name' or a plain string - model_names = [] - for m in models: - if isinstance(m, dict): - model_names.append(m.get('model_name', '')) - else: - model_names.append(str(m)) - return { - 'base_url': url, - 'supported_models': model_names, - } - - try: - return await asyncio.get_event_loop().run_in_executor(None, _query) - except Exception as e: - return {'error': f'Failed to query {url}: {e}'} - - async def _tool_search_datasets(self, query: str, limit: int = 5) -> dict: - """Search ModelScope for datasets.""" - return await self._search_hub('datasets', query, limit) - - async def _tool_search_models(self, query: str, limit: int = 5) -> dict: - """Search ModelScope for models.""" - return await self._search_hub('models', query, limit) - - async def _search_hub(self, resource_type: str, query: str, limit: int) -> dict: - """Unified ModelScope Hub search for models or datasets.""" - - def _search(): - if resource_type == 'datasets': - return self._search_datasets_impl(query, limit) - else: - return self._search_models_impl(query, limit) - - try: - items = await asyncio.get_event_loop().run_in_executor(None, _search) - return {'query': query, 'results': items} - except Exception as e: - return {'error': f'{resource_type.title()} search failed: {e}'} - - @staticmethod - def _search_datasets_impl(query: str, limit: int) -> list[dict]: - """Search datasets via ModelScope SDK (new API).""" - from modelscope.hub.api import HubApi - api = HubApi() - result = api.list_datasets('', search=query, page_size=limit) - datasets = result.get('datasets', []) - return [{'id': d.get('id', ''), 'name': d.get('display_name', d.get('id', ''))} for d in datasets] - - @staticmethod - def _search_models_impl(query: str, limit: int) -> list[dict]: - """Search models via ModelScope HTTP API (SDK doesn't support search).""" - import requests - resp = requests.put( - 'https://modelscope.cn/api/v1/models/', - json={ - 'Name': query, - 'PageSize': limit, - 'PageNumber': 1 - }, - timeout=15, - ) - resp.raise_for_status() - data = resp.json() - if not data.get('Success'): - raise RuntimeError(data.get('Message', 'Unknown error')) - models = data.get('Data', {}).get('Models', []) - return [{ - 'id': f"{m.get('Path', '')}/{m.get('Name', '')}", - 'name': m.get('ChineseName') or m.get('Name', ''), - } for m in models] - - # ── Cluster info ── - - async def _tool_get_cluster_info(self) -> dict: - """Query cluster resources: try Ray first, fall back to nvidia-smi.""" - - def _query(): - # 1. Try connecting to an existing Ray cluster - ray_info = self._try_ray_cluster() - if ray_info is not None: - ray_info['ray_active'] = True - return ray_info - - # 2. Ray not available — fall back to nvidia-smi - nvidia_info = self._try_nvidia_smi() - nvidia_info['ray_active'] = False - nvidia_info['hint'] = ('Ray cluster is not running. To use distributed training, ' - 'start Ray first: `ray start --head --num-gpus=N` or use ' - 'the server mode run.sh script.') - return nvidia_info - - return await asyncio.get_event_loop().run_in_executor(None, _query) - - @staticmethod - def _try_ray_cluster() -> dict | None: - """Attempt to query an existing Ray cluster. Returns None if unavailable.""" - try: - import ray - except ImportError: - return None - - import logging as _logging - - try: - if not ray.is_initialized(): - ray.init( - address='auto', - ignore_reinit_error=True, - _timeout_s=5, - logging_level=_logging.ERROR, - configure_logging=False, - ) - - resources = ray.cluster_resources() - available = ray.available_resources() - nodes = ray.nodes() - gpu_total = resources.get('GPU', 0) - gpu_available = available.get('GPU', 0) - gpu_types = set() - for node in nodes: - for key in node.get('Resources', {}): - if key.startswith('accelerator_type:'): - gpu_types.add(key.split(':', 1)[1]) - return { - 'num_nodes': len([n for n in nodes if n.get('Alive')]), - 'gpu_total': int(gpu_total), - 'gpu_available': int(gpu_available), - 'gpu_types': sorted(gpu_types) if gpu_types else ['unknown'], - 'cpu_total': resources.get('CPU', 0), - 'memory_bytes': resources.get('memory', 0), - } - except Exception: - try: - import ray as _ray - if _ray.is_initialized(): - _ray.shutdown() - except Exception: - pass - return None - - @staticmethod - def _try_nvidia_smi() -> dict: - """Parse nvidia-smi output for local GPU info.""" - import subprocess as _sp - - try: - result = _sp.run( - [ - 'nvidia-smi', '--query-gpu=index,name,memory.total,memory.free,utilization.gpu', - '--format=csv,noheader,nounits' - ], - capture_output=True, - text=True, - timeout=10, - ) - if result.returncode != 0: - return {'error': f'nvidia-smi failed: {result.stderr.strip()}', 'gpu_total': 0} - - gpus = [] - for line in result.stdout.strip().split('\n'): - if not line.strip(): - continue - parts = [p.strip() for p in line.split(',')] - if len(parts) >= 5: - try: - gpus.append({ - 'index': int(parts[0]), - 'name': parts[1], - 'memory_total_mb': int(parts[2]), - 'memory_free_mb': int(parts[3]), - 'utilization_pct': int(parts[4]) if parts[4].isdigit() else 0, - }) - except (ValueError, IndexError): - # Skip lines with unparseable values (e.g. [N/A]) - continue - - gpu_types = sorted({g['name'] for g in gpus}) - return { - 'gpu_total': len(gpus), - 'gpu_available': len([g for g in gpus if g['utilization_pct'] < 10]), - 'gpu_types': gpu_types if gpu_types else ['none'], - 'gpus': gpus, - 'source': 'nvidia-smi', - } - except FileNotFoundError: - return {'error': 'nvidia-smi not found (no NVIDIA GPU or driver not installed)', 'gpu_total': 0} - except Exception as e: - return {'error': f'nvidia-smi query failed: {e}', 'gpu_total': 0} diff --git a/src/twinkle_client/common/serialize.py b/src/twinkle_client/common/serialize.py index 42a27beb..41a56ed7 100644 --- a/src/twinkle_client/common/serialize.py +++ b/src/twinkle_client/common/serialize.py @@ -8,12 +8,7 @@ from twinkle.dataset import DatasetMeta -supported_types = { - DatasetMeta, - LoraConfig, -} - -primitive_types = (str, Number, bool, bytes, type(None)) +primitive_types = (str, Number, bool, type(None)) container_types = (Mapping, list, tuple, set, frozenset) basic_types = (*primitive_types, *container_types) _DATASET_META_FIELDS = {field.name for field in fields(DatasetMeta)} @@ -45,7 +40,9 @@ def _deserialize_data_slice(data_slice): raise ValueError(f'Unsupported data_slice type: {slice_type}') -def serialize_object(obj) -> str: +def serialize_object(obj) -> Any: + if isinstance(obj, (bytes, bytearray, memoryview)): + raise TypeError(f'Unsupported binary object: {type(obj).__name__}') if isinstance(obj, DatasetMeta): data = {name: getattr(obj, name) for name in _DATASET_META_FIELDS} data['data_slice'] = _serialize_data_slice(data.get('data_slice')) diff --git a/src/twinkle_client/http/__init__.py b/src/twinkle_client/http/__init__.py index 9ea02f3b..ab6f48e9 100644 --- a/src/twinkle_client/http/__init__.py +++ b/src/twinkle_client/http/__init__.py @@ -1,6 +1,6 @@ -from .http_utils import http_delete, http_get, http_post, http_post_model -from .utils import (TWINKLE_SERVER_TOKEN, TWINKLE_SERVER_URL, get_api_key, get_base_url, get_request_id, get_session_id, - set_api_key, set_base_url, set_request_id, set_session_id) +from .client import http_delete, http_get, http_post, http_post_model +from .context import (TWINKLE_SERVER_TOKEN, TWINKLE_SERVER_URL, get_api_key, get_base_url, get_request_id, + get_session_id, set_api_key, set_base_url, set_request_id, set_session_id) __all__ = [ 'http_get', diff --git a/src/twinkle_client/http/http_utils.py b/src/twinkle_client/http/client.py similarity index 98% rename from src/twinkle_client/http/http_utils.py rename to src/twinkle_client/http/client.py index 766739e9..6ec5488c 100644 --- a/src/twinkle_client/http/http_utils.py +++ b/src/twinkle_client/http/client.py @@ -2,8 +2,8 @@ from typing import Any, Dict, Optional from twinkle_client.exceptions import TwinkleHTTPError +from .context import get_api_key, get_base_url, get_request_id, get_session_id from .headers import build_routing_headers -from .utils import get_api_key, get_base_url, get_request_id, get_session_id # Single shared HTTP timeout for every client request (was three separate 600s). # Must be <= 120 and strictly greater than the server Long_Poll_Window (default 30), diff --git a/src/twinkle_client/http/utils.py b/src/twinkle_client/http/context.py similarity index 100% rename from src/twinkle_client/http/utils.py rename to src/twinkle_client/http/context.py diff --git a/src/twinkle_client/rollout/multi_turn.py b/src/twinkle_client/rollout/multi_turn.py index 54365cb7..911783a9 100644 --- a/src/twinkle_client/rollout/multi_turn.py +++ b/src/twinkle_client/rollout/multi_turn.py @@ -28,6 +28,16 @@ from twinkle_client.sampler import vLLMSampler +@dataclasses.dataclass +class _RolloutState: + pifs: list[dict[str, Any]] + all_logprobs: list[list[Any]] + stop_reasons: list[str | None] + turns: list[int] + truncated: list[bool] + done: list[bool] + + class ClientMultiTurnRollout: """Agentic multi-turn rollout with tool use, driven over HTTP. @@ -107,159 +117,147 @@ def __call__(self, trajectories: List[Trajectory], **kwargs) -> List[Trajectory] sampling_params = self._as_sampling_params_dict(kwargs.get('sampling_params', self.sampling_params)) tool_managers = self._resolve_tool_managers(kwargs.get('tool_manager', self.tool_manager), n) - # 1. Encode each trajectory once; ``pifs[i]`` is the live per-turn - # state for trajectory ``i``. ``vLLMSampler.sample`` is responsible for - # JSON-serialising the feature (ndarray / tensor -> list) before the - # HTTP POST, so no conversion is needed here. - pifs: List[Dict[str, Any]] = [] - for traj in trajectories: - pif = self.template.encode(traj, add_generation_prompt=True) - pif.setdefault('messages', list(traj.get('messages', []))) - pifs.append(pif) - - all_logprobs: List[List[Any]] = [[] for _ in range(n)] - stop_reasons: List[Optional[str]] = [None] * n - turns: List[int] = [0] * n - truncated: List[bool] = [False] * n - done: List[bool] = [False] * n - + state = self._initialize_state(trajectories) for _ in range(self.max_turns): - active = [i for i in range(n) if not done[i]] + active = [index for index in range(n) if not state.done[index]] if not active: break - # 2. One batched HTTP sample call for all currently-live - # trajectories. No device_mesh / min_batch_size padding: an HTTP - # client has no Ray DP ranks to align against. - # - # Passthrough contract: ``vLLMSampler.sample()`` may raise - # network / timeout / HTTP errors (e.g. requests exceptions). We - # deliberately do NOT wrap this call in try/except -- such errors - # propagate unchanged to the caller so ret/backoff policy stays an - # upstream concern (retry/backoff) and failures are never - # silently swallowed. - batch_pifs = [pifs[i] for i in active] - resps = self.sampler.sample(batch_pifs, sampling_params=sampling_params) - - pending_bridges: List[tuple] = [] # (global_idx, tool_messages) - for local_idx, global_idx in enumerate(active): - turns[global_idx] += 1 - seq = resps[local_idx].sequences[0] - - # ``new_input_feature`` is the running pif for the next round; - # the /twinkle/sample response contract guarantees it is set and - # carries ``input_ids``. A missing feature makes the next round - # impossible, so raise a batch/trajectory-indexed RuntimeError. - if seq.new_input_feature is None or 'input_ids' not in seq.new_input_feature: - raise RuntimeError(f'Sampler returned a sequence without new_input_feature.input_ids at ' - f'batch index {local_idx} (trajectory {global_idx}); ' - f'cannot continue multi-turn.') - - pifs[global_idx] = dict(seq.new_input_feature) - # Per-round logprobs/token alignment guard: each sampled token - # must carry exactly one logprob entry. Mirrors the core-lib - # ``len(seq.logprobs) != len(seq.tokens)`` semantic so client and - # Ray paths cannot drift on this invariant. - if seq.logprobs is not None: - if len(seq.logprobs) != len(seq.tokens): - raise RuntimeError(f'logprobs length ({len(seq.logprobs)}) does not match sampled ' - f'token count ({len(seq.tokens)}) at turn {turns[global_idx]} ' - f'(trajectory {global_idx})') - all_logprobs[global_idx].extend(seq.logprobs) - stop_reasons[global_idx] = seq.stop_reason - - # 3. Termination conditions. - # Cut off at ``max_tokens``: truncated, same as the max_turns and - # length-cap cases below, and same as ``MultiTurnRollout`` and - # ``ApiMultiTurnRollout``. Tool calls in the cut reply are still - # not dispatched. - if seq.stop_reason == 'length': - truncated[global_idx] = True - done[global_idx] = True - continue - - # 3a. Sequence-length cap. - if (self.max_trajectory_tokens is not None - and len(pifs[global_idx].get('input_ids') or []) >= self.max_trajectory_tokens): - truncated[global_idx] = True - done[global_idx] = True - continue - - # 3b. Parse tool calls from the freshly sampled assistant turn. - _msgs = pifs[global_idx].get('messages') or [] - _last_msg = _msgs[-1] if _msgs else None - tool_calls = (_last_msg.get('tool_calls') if isinstance(_last_msg, dict) else None) - if not tool_calls: - tool_calls = self.template.parse_tool_call(seq.decoded or '') - if not tool_calls: - done[global_idx] = True - continue - - # 3c. Hit the turn cap while still wanting to call a tool: force - # truncation. Also covers the ``max_turns == 1`` edge, where - # the very first sampled turn trips this branch. - if turns[global_idx] >= self.max_turns: - truncated[global_idx] = True - stop_reasons[global_idx] = 'max_turns' - done[global_idx] = True - continue - - # 4. Dispatch tools for this trajectory via its ToolManager. - tool_manager = tool_managers[global_idx] - if tool_manager is None: - raise ValueError(f'trajectory {global_idx} produced tool_calls but no tool_manager ' - f'was provided (at construction time or as a per-call kwarg).') - tool_messages = [{ - 'role': 'tool', - 'content': tool_manager(tc), - } for tc in tool_calls] - pending_bridges.append((global_idx, tool_messages)) - - # Stitch bridge tokens (tool turns + next generation prompt) for - # every trajectory with outstanding tool turns. Reuses the shared - # pure function so client and core-lib paths cannot drift. - for global_idx, tool_messages in pending_bridges: - extended = extend_with_bridge(pifs[global_idx], tool_messages, self.template) - if extended is None: - # Trajectory exceeded max_length (truncation strategy 'delete'). - truncated[global_idx] = True - done[global_idx] = True - else: - pifs[global_idx] = extended - - # 4b. Final logprobs/labels alignment guard. For every trajectory that - # collected logprobs, the total logprob count must equal the number - # of trainable positions (labels != -100) in the final pif. This is - # the same invariant grpo._pad_and_align_to_batch relies on; a - # mismatch would silently corrupt GRPO old_logps alignment, so we - # fail loudly with the specific numbers. - for i in range(n): - if not all_logprobs[i]: + # One batched HTTP call for all live trajectories. Network and timeout + # errors intentionally propagate unchanged so retry policy stays upstream. + responses = self.sampler.sample( + [state.pifs[index] for index in active], + sampling_params=sampling_params, + ) + pending_bridges = self._process_responses(active, responses, state, tool_managers) + self._apply_bridges(state, pending_bridges) + + self._validate_logprob_alignment(state.pifs, state.all_logprobs) + return self._build_outputs( + trajectories, + state.pifs, + state.all_logprobs, + state.turns, + state.stop_reasons, + state.truncated, + ) + + # ------------------------------------------------------------------ private + + def _initialize_state(self, trajectories: List[Trajectory]) -> _RolloutState: + pifs: list[dict[str, Any]] = [] + for trajectory in trajectories: + pif = self.template.encode(trajectory, add_generation_prompt=True) + pif.setdefault('messages', list(trajectory.get('messages', []))) + pifs.append(pif) + size = len(trajectories) + return _RolloutState( + pifs=pifs, + all_logprobs=[[] for _ in range(size)], + stop_reasons=[None] * size, + turns=[0] * size, + truncated=[False] * size, + done=[False] * size, + ) + + def _process_responses(self, active, responses, state: _RolloutState, + tool_managers) -> list[tuple[int, list[dict]]]: + pending_bridges = [] + for local_index, global_index in enumerate(active): + sequence = responses[local_index].sequences[0] + tool_messages = self._process_sequence(local_index, global_index, sequence, state, + tool_managers[global_index]) + if tool_messages is not None: + pending_bridges.append((global_index, tool_messages)) + return pending_bridges + + def _process_sequence(self, local_index, global_index, sequence, state: _RolloutState, + tool_manager: ToolManager | None) -> list[dict] | None: + state.turns[global_index] += 1 + if sequence.new_input_feature is None or 'input_ids' not in sequence.new_input_feature: + raise RuntimeError(f'Sampler returned a sequence without new_input_feature.input_ids at ' + f'batch index {local_index} (trajectory {global_index}); ' + f'cannot continue multi-turn.') + + state.pifs[global_index] = dict(sequence.new_input_feature) + if sequence.logprobs is not None: + if len(sequence.logprobs) != len(sequence.tokens): + raise RuntimeError(f'logprobs length ({len(sequence.logprobs)}) does not match sampled ' + f'token count ({len(sequence.tokens)}) at turn {state.turns[global_index]} ' + f'(trajectory {global_index})') + state.all_logprobs[global_index].extend(sequence.logprobs) + state.stop_reasons[global_index] = sequence.stop_reason + + if sequence.stop_reason == 'length' or self._at_token_limit(state.pifs[global_index]): + state.truncated[global_index] = True + state.done[global_index] = True + return None + + messages = state.pifs[global_index].get('messages') or [] + last_message = messages[-1] if messages else None + tool_calls = last_message.get('tool_calls') if isinstance(last_message, dict) else None + tool_calls = tool_calls or self.template.parse_tool_call(sequence.decoded or '') + if not tool_calls: + state.done[global_index] = True + return None + if state.turns[global_index] >= self.max_turns: + state.truncated[global_index] = True + state.stop_reasons[global_index] = 'max_turns' + state.done[global_index] = True + return None + if tool_manager is None: + raise ValueError(f'trajectory {global_index} produced tool_calls but no tool_manager ' + f'was provided (at construction time or as a per-call kwarg).') + return [{'role': 'tool', 'content': tool_manager(tool_call)} for tool_call in tool_calls] + + def _at_token_limit(self, pif: dict[str, Any]) -> bool: + return self.max_trajectory_tokens is not None and len(pif.get('input_ids') or []) >= self.max_trajectory_tokens + + def _apply_bridges(self, state: _RolloutState, pending_bridges: list[tuple[int, list[dict]]]) -> None: + for global_index, tool_messages in pending_bridges: + extended = extend_with_bridge(state.pifs[global_index], tool_messages, self.template) + if extended is None: + state.truncated[global_index] = True + state.done[global_index] = True + else: + state.pifs[global_index] = extended + + @staticmethod + def _validate_logprob_alignment(pifs: List[Dict[str, Any]], all_logprobs: List[List[Any]]) -> None: + """Reject output that would corrupt downstream GRPO old-logprob alignment.""" + for index, logprobs in enumerate(all_logprobs): + if not logprobs: continue - labels_i = pifs[i].get('labels') or [] - trainable_i = sum(1 for label in labels_i if label != -100) - if len(all_logprobs[i]) != trainable_i: - raise RuntimeError(f'logprobs/labels misaligned for trajectory {i}: ' - f'{len(all_logprobs[i])} logprobs vs {trainable_i} ' + labels = pifs[index].get('labels') or [] + trainable = sum(1 for label in labels if label != -100) + if len(logprobs) != trainable: + raise RuntimeError(f'logprobs/labels misaligned for trajectory {index}: ' + f'{len(logprobs)} logprobs vs {trainable} ' f'trainable labels (labels != -100). This invariant is ' f'required by grpo._pad_and_align_to_batch; a mismatch ' f'would silently corrupt GRPO old_logps alignment.') - # 5. Merge pif fields into each trajectory dict at TOP LEVEL, preserving - # input length and order. - outs: List[Trajectory] = [] - for i, traj in enumerate(trajectories): - out = dict(traj) - out.update(pifs[i]) - out['messages'] = list(pifs[i].get('messages') or out.get('messages', [])) - out['logprobs'] = all_logprobs[i] if all_logprobs[i] else None - out['turns'] = turns[i] - out['stop_reason'] = stop_reasons[i] - out['truncated'] = truncated[i] - outs.append(out) - return outs - - # ------------------------------------------------------------------ private + @staticmethod + def _build_outputs( + trajectories: List[Trajectory], + pifs: List[Dict[str, Any]], + all_logprobs: List[List[Any]], + turns: List[int], + stop_reasons: List[Optional[str]], + truncated: List[bool], + ) -> List[Trajectory]: + """Merge final per-trajectory state while preserving input order.""" + outputs: List[Trajectory] = [] + for index, trajectory in enumerate(trajectories): + output = dict(trajectory) + output.update(pifs[index]) + output['messages'] = list(pifs[index].get('messages') or output.get('messages', [])) + output['logprobs'] = all_logprobs[index] or None + output['turns'] = turns[index] + output['stop_reason'] = stop_reasons[index] + output['truncated'] = truncated[index] + outputs.append(output) + return outputs @staticmethod def _as_sampling_params_dict(sampling_params) -> Optional[Dict[str, Any]]: diff --git a/src/twinkle_client/types/data.py b/src/twinkle_client/types/data.py index fb8c6e98..9da24ac3 100644 --- a/src/twinkle_client/types/data.py +++ b/src/twinkle_client/types/data.py @@ -56,19 +56,19 @@ _LEFT_TO_RIGHT = Field(union_mode='left_to_right') -Ints1to2 = Annotated[Union[List[StrictInt], List[List[StrictInt]]], _LEFT_TO_RIGHT] -Ints1to3 = Annotated[Union[List[StrictInt], List[List[StrictInt]], List[List[List[StrictInt]]]], _LEFT_TO_RIGHT] -Ints3 = List[List[List[StrictInt]]] +Ints1to2 = Annotated[Union[list[StrictInt], list[list[StrictInt]]], _LEFT_TO_RIGHT] +Ints1to3 = Annotated[Union[list[StrictInt], list[list[StrictInt]], list[list[list[StrictInt]]]], _LEFT_TO_RIGHT] +Ints3 = list[list[list[StrictInt]]] _Number = Union[StrictInt, float] -Numbers1to2 = Annotated[Union[List[_Number], List[List[_Number]]], _LEFT_TO_RIGHT] -Numbers1to4 = Annotated[Union[List[_Number], List[List[_Number]], List[List[List[_Number]]], - List[List[List[List[_Number]]]]], _LEFT_TO_RIGHT] +Numbers1to2 = Annotated[Union[list[_Number], list[list[_Number]]], _LEFT_TO_RIGHT] +Numbers1to4 = Annotated[Union[list[_Number], list[list[_Number]], list[list[list[_Number]]], + list[list[list[list[_Number]]]]], _LEFT_TO_RIGHT] # Media references travel as strings on the wire (local path, ``http(s)://`` URL, or # a ``data:`` base64 URI). ``PIL.Image`` / raw ``bytes`` / ``np.ndarray`` are valid in # the in-process training path but are not JSON, so they are not declared here. -MediaList = List[str] +MediaList = list[str] # The VLM tensor fields batched by concatenation rather than padding. Declared here # because this module must stay free of Twinkle_Core's heavyweight imports; a @@ -155,7 +155,7 @@ class WireTrajectory(DataModel): # instead of being silently accepted and blowing up inside the backend. Order is # encoded-first, matching ``is_encoded``: a trajectory has neither encoded key, so it # cannot satisfy ``WireInputFeature``'s validator. -WireInputs = Union[List[WireInputFeature], List[WireTrajectory]] +WireInputs = Union[list[WireInputFeature], list[WireTrajectory]] def _as_batch(value: Any) -> Any: diff --git a/src/twinkle_client/types/session.py b/src/twinkle_client/types/session.py index da8795d5..07e940de 100644 --- a/src/twinkle_client/types/session.py +++ b/src/twinkle_client/types/session.py @@ -7,7 +7,7 @@ class CreateSessionRequest(StrictRequest): """Request body for POST /twinkle/create_session.""" - metadata: Optional[Dict[str, Any]] = None + metadata: dict[str, Any] | None = None class CreateSessionResponse(ResponseModel): diff --git a/src/twinkle_client/utils/patch_tinker.py b/src/twinkle_client/utils/patch_tinker.py index 352db28f..5b1bb39c 100644 --- a/src/twinkle_client/utils/patch_tinker.py +++ b/src/twinkle_client/utils/patch_tinker.py @@ -12,8 +12,8 @@ import os from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Union +from twinkle_client.http.context import get_api_key, get_request_id from twinkle_client.http.headers import build_routing_headers -from twinkle_client.http.utils import get_api_key, get_request_id _patched = False _loss_fn_config_patched = False @@ -57,8 +57,8 @@ def _patched_async_tinker_init( if api_key is None: api_key = os.environ.get('TWINKLE_SERVER_TOKEN') if api_key is None: - raise TinkerError('The api_key client option must be set either by passing api_key to the client or by ' - 'setting the TWINKLE_SERVER_TOKEN environment variable') + raise TinkerError('The api_key client option must be set either by passing api_key to the client ' + 'or by setting the TWINKLE_SERVER_TOKEN environment variable') # REMOVED: api_key 'tml-' prefix validation # Original code: # if not api_key.startswith("tml-"): diff --git a/tests/server/lifecycle/test_static_guards.py b/tests/server/lifecycle/test_static_guards.py index 5caef4bc..18f280e8 100644 --- a/tests/server/lifecycle/test_static_guards.py +++ b/tests/server/lifecycle/test_static_guards.py @@ -13,9 +13,8 @@ """ from __future__ import annotations -from pathlib import Path - import pytest +from pathlib import Path _REPO_ROOT = Path(__file__).resolve().parents[3] _SRC = _REPO_ROOT / 'src' / 'twinkle' @@ -43,7 +42,7 @@ def test_deleted_symbol_has_zero_occurrences(symbol): def test_client_http_timeout_bounds(): from twinkle.server.lifecycle.poll_config import long_poll_window - from twinkle_client.http.http_utils import _HTTP_TIMEOUT + from twinkle_client.http.client import _HTTP_TIMEOUT assert _HTTP_TIMEOUT <= 120 assert _HTTP_TIMEOUT > long_poll_window() @@ -68,7 +67,7 @@ def test_task_envelope_has_exactly_one_construction_site(): offenders = [site for site in sites if 'server/lifecycle/envelope.py' not in site] assert offenders == [], ('TaskEnvelope must only be constructed in lifecycle/envelope.py ' - f'(via envelope_from_record); found: {offenders}') + f'(via envelope_from_record); found: {offenders}') assert sites, 'expected to find the construction sites inside envelope.py' @@ -86,12 +85,13 @@ def test_server_task_status_enum_matches_client_literal(): from typing import get_args from twinkle.server.utils.task_queue.types import TaskStatus as ServerTaskStatus - from twinkle_client.types.lifecycle import TaskStatus as WireTaskStatus, TERMINAL_STATUSES + from twinkle_client.types.lifecycle import TERMINAL_STATUSES + from twinkle_client.types.lifecycle import TaskStatus as WireTaskStatus server_values = {member.value for member in ServerTaskStatus} wire_values = set(get_args(WireTaskStatus)) assert server_values == wire_values, (f'task status sets drifted: server-only={server_values - wire_values}, ' - f'wire-only={wire_values - server_values}') + f'wire-only={wire_values - server_values}') assert TERMINAL_STATUSES <= wire_values, 'TERMINAL_STATUSES must be a subset of the declared statuses' diff --git a/tests/server/model/test_twinkle_async_inputs.py b/tests/server/model/test_twinkle_async_inputs.py index 22059725..106d1b04 100644 --- a/tests/server/model/test_twinkle_async_inputs.py +++ b/tests/server/model/test_twinkle_async_inputs.py @@ -5,17 +5,26 @@ from starlette.requests import Request import twinkle_client.types as types +from twinkle.server.model.data_plane_inputs import model_result_rows from twinkle.server.model.twinkle_handlers import _register_twinkle_routes -from twinkle.server.model.utils import model_result_rows def test_model_result_rows_keeps_one_output_row_per_sample() -> None: assert model_result_rows( - {'logps': [[-1.0], [-2.0]], 'loss': 0.25}, + { + 'logps': [[-1.0], [-2.0]], + 'loss': 0.25 + }, batch_size=2, ) == [ - {'logps': [-1.0], 'loss': 0.25}, - {'logps': [-2.0], 'loss': 0.25}, + { + 'logps': [-1.0], + 'loss': 0.25 + }, + { + 'logps': [-2.0], + 'loss': 0.25 + }, ] @@ -29,12 +38,16 @@ def __init__(self): self.data_plane = self self.rows = { 'data-a': [{ - 'train_input': {'input_ids': [index]}, + 'train_input': { + 'input_ids': [index] + }, 'sampled_logprobs': [-0.1], 'advantage': 1.0, } for index in range(4)], 'data-b': [{ - 'train_input': {'input_ids': [index]}, + 'train_input': { + 'input_ids': [index] + }, 'sampled_logprobs': [-0.2], 'advantage': -1.0, } for index in range(4, 8)], @@ -75,9 +88,7 @@ async def test_forward_backward_resolves_multiple_data_refs_and_field_kwargs() - app = FastAPI() _register_twinkle_routes(app, lambda: management) route = next( - route for route in app.routes - if getattr(route, 'path', None) == '/twinkle/forward_backward_from_data_plane' - ) + route for route in app.routes if getattr(route, 'path', None) == '/twinkle/forward_backward_from_data_plane') request = Request({'type': 'http', 'headers': []}) request.state.session_id = 'session' body = types.DataPlaneForwardRequest( @@ -122,9 +133,7 @@ async def test_forward_backward_binds_nested_dpo_ref_logps_without_coercion() -> app = FastAPI() _register_twinkle_routes(app, lambda: management) route = next( - route for route in app.routes - if getattr(route, 'path', None) == '/twinkle/forward_backward_from_data_plane' - ) + route for route in app.routes if getattr(route, 'path', None) == '/twinkle/forward_backward_from_data_plane') request = Request({'type': 'http', 'headers': []}) request.state.session_id = 'session' body = types.DataPlaneForwardRequest( diff --git a/tests/twinkle_client/test_async_components.py b/tests/twinkle_client/test_async_components.py index 5dcab1af..87264dd0 100644 --- a/tests/twinkle_client/test_async_components.py +++ b/tests/twinkle_client/test_async_components.py @@ -47,20 +47,22 @@ def post(url, headers=None, data=None, timeout=None, **_kwargs): def _patch_transport(monkeypatch, calls, result_factory): - import twinkle_client.http.http_utils as http_utils import twinkle_client.http as http_module + import twinkle_client.http.client as http_client monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') - monkeypatch.setattr(http_utils, 'get_base_url', lambda: 'http://server/api/v1') - monkeypatch.setattr(http_utils, 'get_api_key', lambda: 'test-key') - monkeypatch.setattr(http_utils.requests, 'post', _recorder(calls, result_factory)) + monkeypatch.setattr(http_client, 'get_base_url', lambda: 'http://server/api/v1') + monkeypatch.setattr(http_client, 'get_api_key', lambda: 'test-key') + monkeypatch.setattr(http_client.requests, 'post', _recorder(calls, result_factory)) def test_model_forward_backward_sends_multiple_data_refs(monkeypatch) -> None: from twinkle_client.model import multi_lora_transformers as module calls: list = [] - _patch_transport(monkeypatch, calls, - lambda url: {} if url.endswith('/create') else _completed({'result': {'loss': 1.0}})) + _patch_transport(monkeypatch, calls, lambda url: {} + if url.endswith('/create') else _completed({'result': { + 'loss': 1.0 + }})) model = module.MultiLoraTransformersModel('ms://base') model.adapter_name = 'adapter' @@ -132,8 +134,10 @@ def test_model_data_plane_forward_uses_a_separate_api(monkeypatch) -> None: from twinkle_client.model import multi_lora_transformers as module calls: list = [] - _patch_transport(monkeypatch, calls, - lambda url: {} if url.endswith('/create') else _completed({'result': {'value': 1}})) + _patch_transport(monkeypatch, calls, lambda url: {} + if url.endswith('/create') else _completed({'result': { + 'value': 1 + }})) model = module.MultiLoraTransformersModel('ms://base') model.adapter_name = 'adapter' @@ -154,9 +158,8 @@ def test_model_data_plane_forward_only_can_append_selected_outputs(monkeypatch) calls: list = [] # The handler wraps its payload as ``{'result': ...}``, so the stub must too -- # otherwise the test asserts against a reply shape the server never sends. - _patch_transport(monkeypatch, calls, - lambda url: {} if url.endswith('/create') else _completed( - {'result': updated_ref.model_dump()})) + _patch_transport(monkeypatch, calls, lambda url: {} + if url.endswith('/create') else _completed({'result': updated_ref.model_dump()})) model = module.MultiLoraTransformersModel('ms://base') model.adapter_name = 'adapter' @@ -184,13 +187,15 @@ def test_sampler_async_data_plane_path_returns_reference_without_materializing(m kind='rollout', ) calls: list = [] - _patch_transport(monkeypatch, calls, - lambda url: {} if url.endswith('/create') else _completed(output_ref.model_dump())) + _patch_transport(monkeypatch, calls, lambda url: {} + if url.endswith('/create') else _completed(output_ref.model_dump())) sampler = module.vLLMSampler('ms://base') result = asyncio.run(sampler.asample_to_data_plane( - [{'input_ids': [1]}], + [{ + 'input_ids': [1] + }], num_samples=4, group_ids=['group-1'], )) diff --git a/tests/twinkle_client/test_async_rl_workers.py b/tests/twinkle_client/test_async_rl_workers.py index 7a79cb8a..332f20cd 100644 --- a/tests/twinkle_client/test_async_rl_workers.py +++ b/tests/twinkle_client/test_async_rl_workers.py @@ -1,7 +1,6 @@ from __future__ import annotations import asyncio - import pytest from twinkle_client.async_rl import Worker, WorkerPipeline @@ -59,6 +58,7 @@ async def failure(): def test_worker_pipeline_rejects_duplicate_role_names() -> None: + async def noop(): return None diff --git a/tests/twinkle_client/test_auto_agent_tools.py b/tests/twinkle_client/test_auto_agent_tools.py new file mode 100644 index 00000000..a62ebd73 --- /dev/null +++ b/tests/twinkle_client/test_auto_agent_tools.py @@ -0,0 +1,69 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Focused unit contracts for the auto-agent tool dispatcher.""" +from __future__ import annotations + +import json +import pytest + +from twinkle_client.auto.agent.tool_schemas import TOOL_SCHEMAS as DECLARED_TOOL_SCHEMAS +from twinkle_client.auto.agent.tools import TOOL_SCHEMAS, ToolExecutor + + +class _Connection: + current_run_id: str | None = None + + def list_training_runs(self): + return [{'run_id': 'run-1'}] + + +@pytest.mark.asyncio +async def test_execute_dispatches_and_serializes_result(): + result = json.loads(await ToolExecutor(_Connection()).execute('list_training_runs', {})) + assert result == [{'run_id': 'run-1'}] + + +@pytest.mark.asyncio +async def test_execute_reports_unknown_tool_without_raising(): + result = json.loads(await ToolExecutor(_Connection()).execute('missing', {})) + assert result == {'error': 'Unknown tool: missing'} + + +@pytest.mark.asyncio +async def test_execute_turns_handler_exception_into_error(monkeypatch): + executor = ToolExecutor(_Connection()) + + async def fail(): + raise RuntimeError('broken') + + monkeypatch.setattr(executor, '_tool_list_training_runs', fail) + result = json.loads(await executor.execute('list_training_runs', {})) + assert result == {'error': 'list_training_runs failed: broken'} + + +@pytest.mark.asyncio +async def test_search_dispatch_stays_behind_executor(monkeypatch): + executor = ToolExecutor(_Connection()) + monkeypatch.setattr(executor, '_search_datasets_impl', lambda query, limit: [{'id': query, 'limit': limit}]) + result = await executor._tool_search_datasets('demo', limit=2) + assert result == { + 'query': 'demo', + 'results': [{ + 'id': 'demo', + 'limit': 2 + }], + } + + +@pytest.mark.asyncio +async def test_server_health_helper_stays_behind_executor(monkeypatch): + import urllib.request + + monkeypatch.setattr(urllib.request, 'urlopen', lambda *args, **kwargs: object()) + assert await ToolExecutor(_Connection())._check_server_health('http://server') is True + + +def test_tool_schema_names_are_unique_and_dispatchable(): + assert TOOL_SCHEMAS is DECLARED_TOOL_SCHEMAS + names = [item['function']['name'] for item in TOOL_SCHEMAS] + assert len(names) == len(set(names)) + assert all(hasattr(ToolExecutor, f'_tool_{name}') for name in names) diff --git a/tests/twinkle_client/test_client_multi_turn_rollout.py b/tests/twinkle_client/test_client_multi_turn_rollout.py index f19bea04..bd771d80 100644 --- a/tests/twinkle_client/test_client_multi_turn_rollout.py +++ b/tests/twinkle_client/test_client_multi_turn_rollout.py @@ -21,13 +21,12 @@ import copy import json +import pytest import re from collections import defaultdict -from typing import Any, Dict, List, Optional - -import pytest from hypothesis import given, settings from hypothesis import strategies as st +from typing import Any, Dict, List, Optional from twinkle.data_format.sampling import SamplingParams from twinkle_agentic.tools.base import Tool @@ -114,8 +113,7 @@ def __init__(self, tokenizer: FakeTokenizer) -> None: def encode(self, trajectory: Dict[str, Any], add_generation_prompt: bool = False) -> Dict[str, Any]: messages = trajectory.get('messages', []) - s = self.tokenizer.apply_chat_template( - messages, tokenize=False, add_generation_prompt=add_generation_prompt) + s = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=add_generation_prompt) input_ids = self.tokenizer.encode(s, add_special_tokens=False) pif: Dict[str, Any] = dict(trajectory) # preserve top-level fields (incl. _tid) pif['input_ids'] = input_ids @@ -416,8 +414,7 @@ def test_logprobs_align_with_trainable_labels(scripts_spec, max_turns): logprobs = out.get('logprobs') if logprobs: trainable = _count_trainable(out.get('labels') or []) - assert len(logprobs) == trainable, ( - f'logprobs({len(logprobs)}) != trainable labels({trainable})') + assert len(logprobs) == trainable, (f'logprobs({len(logprobs)}) != trainable labels({trainable})') # ============================================================================= @@ -450,8 +447,7 @@ def test_max_turns_one_forces_truncation(logprobs_flags): # Every trajectory emits a tool_call on its first (and only allowed) turn. scripts_spec = [{'num_tools': 3, 'terminal': 'stop', 'logprobs': lp} for lp in logprobs_flags] trajectories, sampler, template = _build_from_scripts(scripts_spec) - rollout = ClientMultiTurnRollout( - sampler=sampler, template=template, tool_manager=_make_tool_manager(), max_turns=1) + rollout = ClientMultiTurnRollout(sampler=sampler, template=template, tool_manager=_make_tool_manager(), max_turns=1) outs = rollout(copy.deepcopy(trajectories)) @@ -475,8 +471,7 @@ def test_length_stop_marks_truncated(logprobs_flags): # very first generation is the one that gets cut. scripts_spec = [{'num_tools': 0, 'terminal': 'length', 'logprobs': lp} for lp in logprobs_flags] trajectories, sampler, template = _build_from_scripts(scripts_spec) - rollout = ClientMultiTurnRollout( - sampler=sampler, template=template, tool_manager=_make_tool_manager(), max_turns=4) + rollout = ClientMultiTurnRollout(sampler=sampler, template=template, tool_manager=_make_tool_manager(), max_turns=4) outs = rollout(copy.deepcopy(trajectories)) @@ -546,11 +541,13 @@ def sample(self, inputs, sampling_params=None, **kwargs): def test_missing_new_input_feature_raises_indexed_runtime_error(): """new_input_feature=None -> RuntimeError naming batch AND trajectory index.""" - trajectories, _script_sampler, template = _build_from_scripts( - [{'num_tools': 0, 'terminal': 'stop', 'logprobs': False}]) + trajectories, _script_sampler, template = _build_from_scripts([{ + 'num_tools': 0, + 'terminal': 'stop', + 'logprobs': False + }]) sampler = _NullFeatureSampler(template) - rollout = ClientMultiTurnRollout( - sampler=sampler, template=template, tool_manager=_make_tool_manager(), max_turns=3) + rollout = ClientMultiTurnRollout(sampler=sampler, template=template, tool_manager=_make_tool_manager(), max_turns=3) with pytest.raises(RuntimeError) as excinfo: rollout(copy.deepcopy(trajectories)) @@ -567,10 +564,8 @@ def test_tool_calls_without_tool_manager_raises_value_error(): """tool_calls produced but tool_manager missing -> ValueError.""" # One tool-call turn then a terminal turn; max_turns=2 so the tool-dispatch # site (not the max_turns truncation edge) is what fails. - trajectories, sampler, template = _build_from_scripts( - [{'num_tools': 1, 'terminal': 'stop', 'logprobs': False}]) - rollout = ClientMultiTurnRollout( - sampler=sampler, template=template, tool_manager=None, max_turns=2) + trajectories, sampler, template = _build_from_scripts([{'num_tools': 1, 'terminal': 'stop', 'logprobs': False}]) + rollout = ClientMultiTurnRollout(sampler=sampler, template=template, tool_manager=None, max_turns=2) with pytest.raises(ValueError) as excinfo: rollout(copy.deepcopy(trajectories)) @@ -582,11 +577,9 @@ def test_tool_calls_without_tool_manager_raises_value_error(): def test_tool_calls_without_tool_manager_via_per_call_kwarg_raises_value_error(): """Passing tool_manager=None as a per-call kwarg also raises at dispatch.""" - trajectories, sampler, template = _build_from_scripts( - [{'num_tools': 1, 'terminal': 'stop', 'logprobs': False}]) + trajectories, sampler, template = _build_from_scripts([{'num_tools': 1, 'terminal': 'stop', 'logprobs': False}]) # Constructed WITH a manager, but the per-call override nulls it out. - rollout = ClientMultiTurnRollout( - sampler=sampler, template=template, tool_manager=_make_tool_manager(), max_turns=2) + rollout = ClientMultiTurnRollout(sampler=sampler, template=template, tool_manager=_make_tool_manager(), max_turns=2) with pytest.raises(ValueError): rollout(copy.deepcopy(trajectories), tool_manager=None) @@ -594,12 +587,14 @@ def test_tool_calls_without_tool_manager_via_per_call_kwarg_raises_value_error() def test_sampler_network_error_propagates_unchanged(): """vLLMSampler.sample() network error propagates unchanged (not swallowed/wrapped).""" - trajectories, _script_sampler, template = _build_from_scripts( - [{'num_tools': 0, 'terminal': 'stop', 'logprobs': False}]) + trajectories, _script_sampler, template = _build_from_scripts([{ + 'num_tools': 0, + 'terminal': 'stop', + 'logprobs': False + }]) sentinel = NetworkError('simulated connection reset by peer') sampler = _NetworkFailingSampler(template, sentinel) - rollout = ClientMultiTurnRollout( - sampler=sampler, template=template, tool_manager=_make_tool_manager(), max_turns=3) + rollout = ClientMultiTurnRollout(sampler=sampler, template=template, tool_manager=_make_tool_manager(), max_turns=3) with pytest.raises(NetworkError) as excinfo: rollout(copy.deepcopy(trajectories)) diff --git a/tests/twinkle_client/test_client_orchestrated_grpo.py b/tests/twinkle_client/test_client_orchestrated_grpo.py index 2302e3bf..20953919 100644 --- a/tests/twinkle_client/test_client_orchestrated_grpo.py +++ b/tests/twinkle_client/test_client_orchestrated_grpo.py @@ -7,10 +7,7 @@ from twinkle_client.types import DataRef - -MODULE_PATH = ( - Path(__file__).parents[2] / 'cookbook' / 'client' / 'async_rl' / 'client_orchestrated_grpo.py' -) +MODULE_PATH = (Path(__file__).parents[2] / 'cookbook' / 'client' / 'async_rl' / 'client_orchestrated_grpo.py') def _load_module(): @@ -57,6 +54,7 @@ async def fake_rollout(_sampler, prompt, policy, _semaphore, _group_id): ) class FakeModel: + def __init__(self): self.saved = [] self.steps = 0 @@ -78,6 +76,7 @@ async def calculate_metric(self, **_kwargs): return {'result': {'loss': 1.0 / self.steps, 'grad_norm': 0.5}} class FakeDataPlane: + def __init__(self): self.released = [] @@ -95,9 +94,21 @@ async def run(): model = FakeModel() data_plane = FakeDataPlane() batches = [ - [{'name': 'p0-g0'}, {'name': 'p0-g1'}], - [{'name': 'p1-g0'}, {'name': 'p1-g1'}], - [{'name': 'p2-g0'}, {'name': 'p2-g1'}], + [{ + 'name': 'p0-g0' + }, { + 'name': 'p0-g1' + }], + [{ + 'name': 'p1-g0' + }, { + 'name': 'p1-g1' + }], + [{ + 'name': 'p2-g0' + }, { + 'name': 'p2-g1' + }], ] await module.run_grpo(batches, model, object(), data_plane) return model, data_plane @@ -152,6 +163,7 @@ async def fake_rollout(_sampler, prompt, _policy, _semaphore, _group_id): monkeypatch.setattr(module, 'GRPOAdvantage', lambda: lambda rewards, **_kwargs: [1.0]) class FakeModel: + def __init__(self): self.saved = [] @@ -169,6 +181,7 @@ async def calculate_metric(self, **_kwargs): return {'result': {'loss': 1.0}} class FakeDataPlane: + async def aget(self, ref, *, fields=None): assert fields == ['decoded'] return [{'decoded': ref.ref_id}] @@ -183,7 +196,13 @@ async def run(): model = FakeModel() try: await module.run_grpo( - [[{'name': 'p0'}], [{'name': 'p1'}], [{'name': 'p2'}]], + [[{ + 'name': 'p0' + }], [{ + 'name': 'p1' + }], [{ + 'name': 'p2' + }]], model, object(), FakeDataPlane(), diff --git a/tests/twinkle_client/test_data_plane_async.py b/tests/twinkle_client/test_data_plane_async.py index fd616f34..5a0559a1 100644 --- a/tests/twinkle_client/test_data_plane_async.py +++ b/tests/twinkle_client/test_data_plane_async.py @@ -2,9 +2,8 @@ from __future__ import annotations import asyncio -import threading - import pytest +import threading from twinkle_client.data_plane import DataPlaneClient from twinkle_client.types import DataRef, DataRowsResponse @@ -46,9 +45,13 @@ async def run(): asyncio.run(run()) assert [call[:-1] for call in calls] == [ - ('put', [{'value': 1}], 'rollout'), + ('put', [{ + 'value': 1 + }], 'rollout'), ('get', original_ref, ['value']), - ('append', original_ref, [{'value': 2}]), + ('append', original_ref, [{ + 'value': 2 + }]), ('release', appended_ref), ] assert all(call[-1] != caller_thread for call in calls) @@ -96,7 +99,11 @@ async def run(): asyncio.run(run()) assert calls == [ - ('put', [{'value': 1}], 'data', tags), + ('put', [{ + 'value': 1 + }], 'data', tags), ('get_batch', ref, None), - ('append', ref, [{'reward': 1.0}], tags), + ('append', ref, [{ + 'reward': 1.0 + }], tags), ] diff --git a/tests/twinkle_client/test_error_parsing.py b/tests/twinkle_client/test_error_parsing.py index 7a63dc61..9c322290 100644 --- a/tests/twinkle_client/test_error_parsing.py +++ b/tests/twinkle_client/test_error_parsing.py @@ -6,7 +6,7 @@ import requests from twinkle_client.exceptions import TwinkleHTTPError -from twinkle_client.http.http_utils import _handle_response +from twinkle_client.http.client import _handle_response class _Resp: @@ -30,7 +30,7 @@ def test_structured_error_reads_top_level_fields(): resp = _Resp(422, body={'error': 'bad input', 'category': 'user', 'error_code': 422, 'request_id': 'req-7'}) with pytest.raises(TwinkleHTTPError) as exc: _handle_response(resp) - assert isinstance(exc.value, requests.HTTPError) # R3#8: existing except clauses keep working + assert isinstance(exc.value, requests.HTTPError) # R3#8: existing except clauses keep working assert exc.value.status_code == 422 assert exc.value.error_code == 422 assert exc.value.category == 'user' diff --git a/tests/twinkle_client/test_future_layer.py b/tests/twinkle_client/test_future_layer.py index db0ff132..d8cf1264 100644 --- a/tests/twinkle_client/test_future_layer.py +++ b/tests/twinkle_client/test_future_layer.py @@ -41,6 +41,7 @@ def _running(): def test_terminal_submit_issues_no_retrieve(monkeypatch): """R8#1: a task terminal in the submit envelope makes zero retrieve calls.""" + def _boom(_request_id): raise AssertionError('retrieve must not be called for a terminal submit') @@ -58,7 +59,7 @@ def test_terminal_submit_failure_raises_taskfailed_with_payload(monkeypatch): assert exc.value.category == 'server' assert exc.value.request_id == 'r' assert exc.value.error_code == 500 - assert not isinstance(exc.value, requests.HTTPError) # R3#10 + assert not isinstance(exc.value, requests.HTTPError) # R3#10 def test_model_cls_none_returns_none_result(monkeypatch): @@ -76,6 +77,7 @@ def test_non_terminal_submit_polls_until_terminal(monkeypatch): def test_404_is_bounded_then_raises_record_lost(monkeypatch): + def _always_404(_request_id): e = requests.HTTPError('404') e.status_code = 404 @@ -87,7 +89,7 @@ def _always_404(_request_id): def test_transport_5xx_is_bounded_then_reraises(monkeypatch): - monkeypatch.setattr(_future.time, 'sleep', lambda _s: None) # no real backoff sleeps + monkeypatch.setattr(_future.time, 'sleep', lambda _s: None) # no real backoff sleeps def _always_503(_request_id): e = requests.HTTPError('503') @@ -100,6 +102,7 @@ def _always_503(_request_id): def test_non_retryable_4xx_reraises_immediately(monkeypatch): + def _400(_request_id): e = requests.HTTPError('400') e.status_code = 400 @@ -124,12 +127,12 @@ def test_success_resets_both_retry_counters(monkeypatch): def _mixed(_request_id): seq.append(1) n = len(seq) - if n in (1, 2, 4, 5): # 404s interleaved with a success at n==3 + if n in (1, 2, 4, 5): # 404s interleaved with a success at n==3 e = requests.HTTPError('404') e.status_code = 404 raise e if n == 3: - return _running() # success resets not_found_count + return _running() # success resets not_found_count return _completed({'done': True}) monkeypatch.setattr(_future, '_post_retrieve', _mixed) diff --git a/tests/twinkle_client/test_request_builder.py b/tests/twinkle_client/test_request_builder.py index c65d9d1d..6d743e66 100644 --- a/tests/twinkle_client/test_request_builder.py +++ b/tests/twinkle_client/test_request_builder.py @@ -12,6 +12,7 @@ from typing import Dict from twinkle_client._request_builder import build_request, request_json, to_wire_value +from twinkle_client.common.serialize import serialize_object from twinkle_client.exceptions import TwinkleClientValidationError from twinkle_client.types import model as model_types from twinkle_client.types.base import StrictRequest, passthrough @@ -20,8 +21,8 @@ @pytest.fixture(autouse=True) def no_network(monkeypatch): """Any HTTP call in this module is a bug in the code under test.""" - import twinkle_client.http.http_utils as http_utils - monkeypatch.setattr(http_utils.requests, 'post', + import twinkle_client.http.client as http_client + monkeypatch.setattr(http_client.requests, 'post', lambda *a, **k: pytest.fail('a Level 0 failure must not produce a request')) @@ -40,7 +41,9 @@ def test_an_undeclared_name_goes_to_the_single_passthrough_region(): """Public signatures stay ``**kwargs``; only the wire shape changes.""" body = build_request( model_types.ForwardBackwardTaskRequest, - inputs=[{'input_ids': [1]}], + inputs=[{ + 'input_ids': [1] + }], adapter_name='a', advantages=[0.5], old_logps=[[-1.0]]) @@ -74,7 +77,11 @@ def test_a_model_without_a_region_rejects_an_unknown_name(): def test_an_explicit_region_and_a_routed_key_are_merged(): body = build_request( - model_types.SetLossRequest, loss_cls='DPOLoss', adapter_name='a', init_kwargs={'beta': 0.1}, loss_type='sigmoid') + model_types.SetLossRequest, + loss_cls='DPOLoss', + adapter_name='a', + init_kwargs={'beta': 0.1}, + loss_type='sigmoid') assert body.init_kwargs == {'beta': 0.1, 'loss_type': 'sigmoid'} @@ -140,7 +147,14 @@ def test_a_lora_config_is_serialized_to_the_form_the_server_decodes(): assert isinstance(wire, str) and 'LoraConfig' in wire +def test_binary_values_are_rejected_by_legacy_serializer(): + for value in (b'raw', bytearray(b'raw'), memoryview(b'raw')): + with pytest.raises(TypeError, match='Unsupported binary object'): + serialize_object(value) + + def test_a_component_handle_is_sent_as_its_id(): + class _Handle: processor_id = 'pid:abc' diff --git a/tests/twinkle_client/test_types_contract.py b/tests/twinkle_client/test_types_contract.py index 3b048c61..53ab20e7 100644 --- a/tests/twinkle_client/test_types_contract.py +++ b/tests/twinkle_client/test_types_contract.py @@ -44,8 +44,7 @@ def test_queue_state_literal_matches_server_enum(): literal_values = set(typing.get_args(QueueStateLiteral)) enum_values = {state.value for state in QueueState} - assert literal_values == enum_values, ( - f'QueueStateLiteral {literal_values} != QueueState {enum_values}') + assert literal_values == enum_values, (f'QueueStateLiteral {literal_values} != QueueState {enum_values}') def _origin(module: str | None) -> str | None: @@ -83,13 +82,10 @@ def _binding_collisions(tree: ast.AST) -> set[str]: def test_public_name_overlap_does_not_grow(): import tinker.types + import twinkle_client.types - overlap = { - name - for name in set(dir(tinker.types)) & set(dir(twinkle_client.types)) - if not name.startswith('_') - } + overlap = {name for name in set(dir(tinker.types)) & set(dir(twinkle_client.types)) if not name.startswith('_')} assert overlap == _LEGACY_PUBLIC_NAME_OVERLAP @@ -100,6 +96,5 @@ def test_no_tinker_twinkle_same_name_binding(): collisions = _binding_collisions(tree) if collisions: offenders[str(path.relative_to(_TWINKLE_SRC))] = collisions - assert not offenders, ( - 'tinker and twinkle_client types bound to the same local name (alias tinker ' - f'to disambiguate): {offenders}') + assert not offenders, ('tinker and twinkle_client types bound to the same local name (alias tinker ' + f'to disambiguate): {offenders}') From e03736543d7871e88af2613d3fdb51f4a1b9a19a Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Fri, 18 Sep 2026 15:44:02 +0000 Subject: [PATCH 22/32] update client architecture --- cookbook/client/twinkle/dpo.py | 26 +- cookbook/client/twinkle/embedding.py | 12 +- cookbook/client/twinkle/multi_modal.py | 28 +- cookbook/client/twinkle/multi_turn_rollout.py | 34 +- cookbook/client/twinkle/sample.py | 11 +- cookbook/client/twinkle/self_cognition.py | 14 +- cookbook/client/twinkle/short_math_grpo.py | 48 +- cookbook/client/twinkle/upload_to_hub.py | 9 +- pyproject.toml | 1 + src/twinkle/server/gateway/services.py | 63 +++ src/twinkle/server/gateway/tinker_handlers.py | 61 +-- .../server/gateway/twinkle_handlers.py | 62 ++- src/twinkle_client/__init__.py | 5 +- src/twinkle_client/_future.py | 48 +- src/twinkle_client/common/component_rpc.py | 60 ++- src/twinkle_client/data_plane.py | 18 +- src/twinkle_client/dataloader/__init__.py | 2 + src/twinkle_client/dataloader/dataloader.py | 36 +- src/twinkle_client/dataset/__init__.py | 2 + src/twinkle_client/dataset/base.py | 62 ++- .../dataset/iterable_dataset.py | 27 +- .../dataset/iterable_packing_dataset.py | 34 +- src/twinkle_client/dataset/lazy_dataset.py | 46 +- src/twinkle_client/dataset/packing_dataset.py | 28 +- src/twinkle_client/http/__init__.py | 18 +- src/twinkle_client/http/client.py | 415 ++++++++++-------- src/twinkle_client/http/context.py | 104 +++-- src/twinkle_client/manager.py | 215 +++++---- src/twinkle_client/model/__init__.py | 2 + .../model/multi_lora_transformers.py | 99 +++-- src/twinkle_client/processor/__init__.py | 2 + src/twinkle_client/processor/base.py | 23 +- src/twinkle_client/py.typed | 1 + src/twinkle_client/sampler/__init__.py | 2 + src/twinkle_client/sampler/vllm_sampler.py | 49 +-- src/twinkle_client/types/__init__.py | 129 +++++- src/twinkle_client/types/server.py | 21 +- .../test_client_architecture_imports.py | 43 ++ tests/server/test_gateway_services.py | 93 ++++ tests/twinkle_client/test_async_components.py | 18 +- tests/twinkle_client/test_future_layer.py | 18 +- tests/twinkle_client/test_transport.py | 149 +++++++ tests/twinkle_client/test_types_contract.py | 26 ++ 43 files changed, 1404 insertions(+), 760 deletions(-) create mode 100644 src/twinkle/server/gateway/services.py create mode 100644 src/twinkle_client/py.typed create mode 100644 tests/server/static/test_client_architecture_imports.py create mode 100644 tests/server/test_gateway_services.py create mode 100644 tests/twinkle_client/test_transport.py diff --git a/cookbook/client/twinkle/dpo.py b/cookbook/client/twinkle/dpo.py index b7fedcd8..a9279813 100644 --- a/cookbook/client/twinkle/dpo.py +++ b/cookbook/client/twinkle/dpo.py @@ -6,24 +6,18 @@ # Step 1: Load environment variables from a .env file (e.g., API tokens) import dotenv -import os -from typing import Any, Dict, List - -dotenv.load_dotenv('.env') import numpy as np +import os import torch from peft import LoraConfig +from typing import Any, Dict, List -from twinkle import get_logger -from twinkle.dataset import Dataset, DatasetMeta -from twinkle_client import init_twinkle_client +from twinkle import get_logger, init_twinkle_client from twinkle.dataloader import DataLoader -from twinkle_client.model import MultiLoraTransformersModel -from twinkle.loss import DPOLoss -from twinkle.metric import DPOMetric +from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import EmojiDPOProcessor -from twinkle.processor import InputProcessor +dotenv.load_dotenv('.env') logger = get_logger() # Configuration (direct values, not from env) @@ -68,11 +62,9 @@ def create_dpo_dataset(): dataset = Dataset(DatasetMeta(dataset_id, data_slice=range(100))) dataset.set_template('Qwen3_5Template', model_id=f'ms://{base_model}', max_length=max_length) dataset.map( - EmojiDPOProcessor, - init_args={ + EmojiDPOProcessor, init_args={ 'system': system_prompt, - } - ) + }) # DPO preprocessor returns {'positive': [...], 'negative': [...]} # batch_encode handles this format automatically dataset.encode() @@ -121,7 +113,7 @@ def train(): # Step 5: Configure the model # Create a multi-LoRA Transformers model pointing to the base model on ModelScope - model = MultiLoraTransformersModel(model_id=f'ms://{base_model}') + model = client.model(f'ms://{base_model}') # Define LoRA configuration: apply low-rank adapters to all linear layers lora_config = LoraConfig( @@ -162,7 +154,7 @@ def train(): optim_step = 0 max_steps = len(dataloader) logger.info(f'Starting LoRA DPO training: loss_type={loss_type}, beta={dpo_beta}, lr={learning_rate}') - logger.info(f'Using base model (disable_lora=True) as reference model') + logger.info('Using base model (disable_lora=True) as reference model') for batch in dataloader: # batch is List[Dict] with 'positive' and 'negative' keys diff --git a/cookbook/client/twinkle/embedding.py b/cookbook/client/twinkle/embedding.py index 304a321f..d020e8a8 100644 --- a/cookbook/client/twinkle/embedding.py +++ b/cookbook/client/twinkle/embedding.py @@ -24,18 +24,14 @@ # megatron mrope model gets valid positions (transformers derives them internally). import dotenv - -dotenv.load_dotenv('.env') - import os -from typing import Any, Dict, List - from peft import LoraConfig +from typing import Any, Dict, List from twinkle import get_logger, init_twinkle_client from twinkle.template import Qwen3_5Template -from twinkle_client.model import MultiLoraTransformersModel +dotenv.load_dotenv('.env') logger = get_logger() # ========== Configuration ========== @@ -84,13 +80,13 @@ def build_minibatch(tokenizer) -> List[Dict[str, Any]]: def train(): # Step 1: connect to the running Twinkle server. - init_twinkle_client( + client = init_twinkle_client( base_url=os.environ.get('TWINKLE_SERVER_URL', 'http://localhost:8000'), api_key=os.environ.get('TWINKLE_SERVER_TOKEN', 'EMPTY_TOKEN'), ) # Step 2: build the client model with a fresh LoRA adapter. - model = MultiLoraTransformersModel(model_id=MODEL_ID) + model = client.model(MODEL_ID) model.add_adapter_to_model(ADAPTER_NAME, LoraConfig(target_modules='all-linear')) model.set_template('Qwen3_5Template', model_id=MODEL_ID) diff --git a/cookbook/client/twinkle/multi_modal.py b/cookbook/client/twinkle/multi_modal.py index 3a5a45b6..6783244b 100644 --- a/cookbook/client/twinkle/multi_modal.py +++ b/cookbook/client/twinkle/multi_modal.py @@ -6,22 +6,18 @@ # Step 1: Load environment variables from a .env file (e.g., API tokens) import dotenv -import os -from twinkle.data_format import Trajectory, Message -from twinkle.preprocessor import Preprocessor - -dotenv.load_dotenv('.env') import numpy as np +import os import torch from peft import LoraConfig -from twinkle import get_logger -from twinkle.dataset import DatasetMeta -from twinkle_client import init_twinkle_client +from twinkle import get_logger, init_twinkle_client +from twinkle.data_format import Message, Trajectory from twinkle.dataloader import DataLoader -from twinkle.dataset import LazyDataset -from twinkle_client.model import MultiLoraTransformersModel +from twinkle.dataset import DatasetMeta, LazyDataset +from twinkle.preprocessor import Preprocessor +dotenv.load_dotenv('.env') logger = get_logger() base_model = os.environ.get('TWINKLE_MODEL_ID', 'Qwen/Qwen3.5-4B') @@ -58,12 +54,10 @@ def __call__(self, rows): return rows def preprocess(self, row) -> Trajectory: - return Trajectory( - messages=[ - Message(role='user', content='Using LaTeX to perform OCR on the image.', images=[row['image']]), - Message(role='assistant', content=row['text']), - ] - ) + return Trajectory(messages=[ + Message(role='user', content='Using LaTeX to perform OCR on the image.', images=[row['image']]), + Message(role='assistant', content=row['text']), + ]) def train(): @@ -87,7 +81,7 @@ def train(): # Step 5: Configure the model # Create a multi-LoRA Transformers model pointing to the base model on ModelScope - model = MultiLoraTransformersModel(model_id=f'ms://{base_model}') + model = client.model(f'ms://{base_model}') # Define LoRA configuration: apply low-rank adapters to all linear layers lora_config = LoraConfig(target_modules='all-linear') diff --git a/cookbook/client/twinkle/multi_turn_rollout.py b/cookbook/client/twinkle/multi_turn_rollout.py index 199dca38..6325976b 100644 --- a/cookbook/client/twinkle/multi_turn_rollout.py +++ b/cookbook/client/twinkle/multi_turn_rollout.py @@ -21,11 +21,10 @@ # ``model.save(is_sampler=True)`` and point the sampler at the saved adapter; that # sync is intentionally omitted here to keep the rollout example focused. -import os -from typing import Any, Dict, List, Tuple - import dotenv +import os from peft import LoraConfig +from typing import Any, Dict, List, Tuple from twinkle import get_logger, init_twinkle_client from twinkle.advantage import GRPOAdvantage @@ -33,9 +32,7 @@ from twinkle.template import Qwen3_5Template from twinkle_agentic.envs import EnvTool, OpenEnv from twinkle_agentic.tools.tool_manager import ToolManager -from twinkle_client.model import MultiLoraTransformersModel from twinkle_client.rollout import ClientMultiTurnRollout -from twinkle_client.sampler import vLLMSampler dotenv.load_dotenv('.env') @@ -45,7 +42,7 @@ BASE_MODEL = os.environ.get('TWINKLE_MODEL_ID', 'Qwen/Qwen3.5-4B') MODEL_ID = f'ms://{BASE_MODEL}' ADAPTER_NAME = 'default' -NUM_GENERATIONS = 2 # GRPO group size (rollout runs num_samples=1 per trajectory) +NUM_GENERATIONS = 2 # GRPO group size (rollout runs num_samples=1 per trajectory) BATCH_SIZE = 2 MAX_NEW_TOKENS = 512 MAX_TURNS = 4 @@ -81,7 +78,8 @@ Your goal is to win the game by getting as close to 21 as possible without going over. -Use the `play` tool to choose either `hit` or `stand`. Reason briefly before each action. Once the environment reports that the game is over, give a short final answer without calling another tool.""" +Use the `play` tool to choose either `hit` or `stand`. Reason briefly before each action. +Once the environment reports that the game is over, give a short final answer without calling another tool.""" def blackjack_action_mapper(tool_name: str, arguments: Dict[str, Any]) -> Dict[str, Any]: @@ -105,8 +103,7 @@ def create_env_tool(env: OpenEnv) -> EnvTool: def prepare_trajectories( - n_trajectories: int, -) -> Tuple[List[Dict[str, Any]], List[ToolManager], List[List[EnvTool]], List[OpenEnv]]: + n_trajectories: int, ) -> Tuple[List[Dict[str, Any]], List[ToolManager], List[List[EnvTool]], List[OpenEnv]]: """Create and reset one independent OpenEnv instance per trajectory.""" trajectories = [] tool_managers = [] @@ -127,10 +124,17 @@ def prepare_trajectories( tool_manager = ToolManager(env_tools) trajectories.append({ 'messages': [ - {'role': 'system', 'content': SYSTEM_PROMPT}, - {'role': 'user', 'content': initial_observation}, + { + 'role': 'system', + 'content': SYSTEM_PROMPT + }, + { + 'role': 'user', + 'content': initial_observation + }, ], - 'tools': tool_manager.tool_infos(), + 'tools': + tool_manager.tool_infos(), }) tool_managers.append(tool_manager) env_tools_list.append(env_tools) @@ -149,13 +153,13 @@ def extract_rewards(env_tools_list: List[List[EnvTool]]) -> List[float]: def train(): # Step 1: connect to the running Twinkle server. - init_twinkle_client( + client = init_twinkle_client( base_url=os.environ.get('TWINKLE_SERVER_URL', 'http://localhost:8000'), api_key=os.environ.get('TWINKLE_SERVER_TOKEN', 'EMPTY_TOKEN'), ) # Step 2: training model (GRPO), mirroring the ray-local example's config. - model = MultiLoraTransformersModel(model_id=MODEL_ID) + model = client.model(MODEL_ID) model.add_adapter_to_model(ADAPTER_NAME, LoraConfig(target_modules='all-linear', r=16, lora_alpha=32)) model.set_loss('GRPOLoss', epsilon=0.2) model.set_optimizer('Adam', lr=LEARNING_RATE) @@ -163,7 +167,7 @@ def train(): model.set_template('Qwen3_5Template', model_id=MODEL_ID, enable_thinking=False) # Step 3: client sampler (HTTP). - sampler = vLLMSampler(model_id=MODEL_ID) + sampler = client.sampler(MODEL_ID) sampler.set_template('Qwen3_5Template', model_id=MODEL_ID, enable_thinking=False) # Step 4: multi-turn rollout. Each call receives trajectory-bound ToolManagers diff --git a/cookbook/client/twinkle/sample.py b/cookbook/client/twinkle/sample.py index 5bbfd424..be6471de 100644 --- a/cookbook/client/twinkle/sample.py +++ b/cookbook/client/twinkle/sample.py @@ -10,16 +10,12 @@ # Step 1: Load environment variables from a .env file (e.g., API tokens) import dotenv - -dotenv.load_dotenv('.env') - import os from transformers import AutoTokenizer -from twinkle import get_logger -from twinkle_client import init_twinkle_client -from twinkle_client.sampler import vLLMSampler +from twinkle import get_logger, init_twinkle_client +dotenv.load_dotenv('.env') logger = get_logger() MODEL_ID = os.environ.get('TWINKLE_MODEL_ID', 'Qwen/Qwen3.5-4B') @@ -31,6 +27,7 @@ # Example: ADAPTER_URI = 'twinkle://20260301_142318-Qwen_Qwen3-4B-199d2cdb/weights/twinkle-lora-0' + def sample(): # Step 2: Initialize the Twinkle client to communicate with the remote server. client = init_twinkle_client( @@ -39,7 +36,7 @@ def sample(): ) # Step 3: Create the sampler client pointing to the model on the server - sampler = vLLMSampler(model_id=MODEL_ID) + sampler = client.sampler(MODEL_ID) # Step 4: Set the chat template so the sampler can encode Trajectory inputs sampler.set_template('Qwen3_5Template', model_id=MODEL_ID) diff --git a/cookbook/client/twinkle/self_cognition.py b/cookbook/client/twinkle/self_cognition.py index f5250b29..a7747389 100644 --- a/cookbook/client/twinkle/self_cognition.py +++ b/cookbook/client/twinkle/self_cognition.py @@ -6,19 +6,14 @@ # Step 1: Load environment variables from a .env file (e.g., API tokens) import dotenv - -dotenv.load_dotenv('.env') - import os from peft import LoraConfig -from twinkle import get_logger -from twinkle.dataset import DatasetMeta -from twinkle import init_twinkle_client +from twinkle import get_logger, init_twinkle_client from twinkle.dataloader import DataLoader -from twinkle.dataset import Dataset -from twinkle_client.model import MultiLoraTransformersModel +from twinkle.dataset import Dataset, DatasetMeta +dotenv.load_dotenv('.env') logger = get_logger() base_model = os.environ.get('TWINKLE_MODEL_ID', 'Qwen/Qwen3.5-4B') @@ -26,7 +21,6 @@ api_key = os.environ.get('TWINKLE_SERVER_TOKEN', 'EMPTY_TOKEN') save_dir = '/tmp/twinkle_sft_output' - # Step 2: Initialize the Twinkle client to communicate with the remote server. # - base_url: the address of the running Twinkle server # - api_key: authentication token (loaded from environment variable) @@ -74,7 +68,7 @@ def train(): # Step 5: Configure the model # Create a multi-LoRA Transformers model pointing to the base model on ModelScope - model = MultiLoraTransformersModel(model_id=f'ms://{base_model}') + model = client.model(f'ms://{base_model}') # Define LoRA configuration: apply low-rank adapters to all linear layers lora_config = LoraConfig(target_modules='all-linear') diff --git a/cookbook/client/twinkle/short_math_grpo.py b/cookbook/client/twinkle/short_math_grpo.py index 1e90d38c..d58e5e9c 100644 --- a/cookbook/client/twinkle/short_math_grpo.py +++ b/cookbook/client/twinkle/short_math_grpo.py @@ -20,30 +20,23 @@ # Requires both model and sampler services to be configured. import dotenv - -dotenv.load_dotenv('.env') - import gc import os import re -from peft import LoraConfig -from typing import List, Tuple, Dict, Any - import swanlab +from peft import LoraConfig +from typing import Any, Dict, List, Tuple -from twinkle import get_logger -from twinkle.reward import GSM8KAccuracyReward -from twinkle.reward.base import Reward +from twinkle import get_logger, init_twinkle_client from twinkle.advantage import GRPOAdvantage -from twinkle.dataset import DatasetMeta -from twinkle.metric import CompletionRewardMetric -from twinkle import init_twinkle_client from twinkle.dataloader import DataLoader -from twinkle.dataset import Dataset +from twinkle.dataset import Dataset, DatasetMeta +from twinkle.metric import CompletionRewardMetric from twinkle.preprocessor.llm import GSM8KProcessor -from twinkle_client.model import MultiLoraTransformersModel -from twinkle_client.sampler import vLLMSampler +from twinkle.reward import GSM8KAccuracyReward +from twinkle.reward.base import Reward +dotenv.load_dotenv('.env') logger = get_logger() @@ -64,10 +57,7 @@ def __call__(self, trajectories: List[Dict[str, Any]], **kwargs) -> List[float]: completion = msg.get('content', '') break - has_answer = bool( - re.search(r'\\boxed\{[^}]+\}', completion) - or re.search(r'####\s*[\-\d,\.]+', completion) - ) + has_answer = bool(re.search(r'\\boxed\{[^}]+\}', completion) or re.search(r'####\s*[\-\d,\.]+', completion)) if not has_answer: rewards.append(0.0) @@ -79,6 +69,7 @@ def __call__(self, trajectories: List[Dict[str, Any]], **kwargs) -> List[float]: rewards.append(max(0.0, 1.0 - (length - 200) / 3000)) return rewards + # ========== Configuration ========== BASE_MODEL = os.environ.get('TWINKLE_MODEL_ID', 'Qwen/Qwen3.5-4B') MODEL_ID = f'ms://{BASE_MODEL}' @@ -96,20 +87,20 @@ def __call__(self, trajectories: List[Dict[str, Any]], **kwargs) -> List[float]: SWANLAB_PROJECT = 'twinkle-grpo' SWANLAB_EXPERIMENT_NAME = 'short-math-grpo' - SYSTEM_PROMPT = ('You are a helpful math assistant. Solve the problem with minimal but correct reasoning ' 'and put your final answer within \\boxed{}.') + def create_gsm8k_dataset(): - dataset = Dataset(DatasetMeta('ms://modelscope/gsm8k', subset_name='main', split='train', data_slice=range(DATA_NUM))) + dataset = Dataset( + DatasetMeta('ms://modelscope/gsm8k', subset_name='main', split='train', data_slice=range(DATA_NUM))) dataset.set_template('Qwen3_5Template', model_id=MODEL_ID, max_length=2048, enable_thinking=False) dataset.map(GSM8KProcessor(system=SYSTEM_PROMPT)) dataset.encode(add_generation_prompt=True) return dataset -def compute_rewards( - trajectories: List[Dict[str, Any]], -) -> Tuple[List[float], List[float], List[float]]: + +def compute_rewards(trajectories: List[Dict[str, Any]], ) -> Tuple[List[float], List[float], List[float]]: accuracy_reward_fn = GSM8KAccuracyReward() brevity_reward_fn = GSM8KBrevityReward() @@ -151,7 +142,7 @@ def train(): dataloader = DataLoader(dataset=dataset, batch_size=BATCH_SIZE, num_workers=0) # Step 3: Configure the training model - model = MultiLoraTransformersModel(model_id=MODEL_ID) + model = client.model(MODEL_ID) lora_config = LoraConfig( target_modules='all-linear', @@ -182,7 +173,7 @@ def train(): model.set_template('Qwen3_5Template', model_id=MODEL_ID) # Step 4: Configure the sampler - sampler = vLLMSampler(model_id=MODEL_ID) + sampler = client.sampler(MODEL_ID) sampler.set_template('Qwen3_5Template', model_id=MODEL_ID) # Step 5: Setup metrics and advantage function @@ -241,9 +232,7 @@ def train(): # ========== 3. Compute rewards ========== - total_rewards, brevity_rewards, accuracy_rewards = compute_rewards( - all_input_data - ) + total_rewards, brevity_rewards, accuracy_rewards = compute_rewards(all_input_data) metrics.accumulate( completion_lengths=all_completion_lengths, rewards={ @@ -253,7 +242,6 @@ def train(): }, ) - # ========== 4. Compute advantages ========== advantages = advantage_fn( total_rewards, diff --git a/cookbook/client/twinkle/upload_to_hub.py b/cookbook/client/twinkle/upload_to_hub.py index f2303622..b884fb73 100644 --- a/cookbook/client/twinkle/upload_to_hub.py +++ b/cookbook/client/twinkle/upload_to_hub.py @@ -17,14 +17,11 @@ # - A ModelScope API token with write access to the target repository import dotenv - -dotenv.load_dotenv('.env') - import os from twinkle import get_logger, init_twinkle_client -from twinkle_client.model import MultiLoraTransformersModel +dotenv.load_dotenv('.env') logger = get_logger() # ── Configuration ───────────────────────────────────────────────────────────── @@ -45,10 +42,10 @@ def upload(): # Step 1: Initialize the Twinkle client - init_twinkle_client(base_url=base_url, api_key=api_key) + client = init_twinkle_client(base_url=base_url, api_key=api_key) # Step 2: Create the model client (no training state needed for upload) - model = MultiLoraTransformersModel(model_id=f'ms://{base_model}') + model = client.model(f'ms://{base_model}') # Step 3: Upload checkpoint to ModelScope Hub. # The client polls for completion automatically; progress is printed to stdout. diff --git a/pyproject.toml b/pyproject.toml index 68500f29..32ea8e64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,5 +88,6 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] +"twinkle_client" = ["py.typed"] "twinkle_client.skills.bundled" = ["*.md"] "twinkle.kernel.ops.dsv4_sas_li.aclnn" = ["*.h", "*.cpp", "**/*.cpp"] diff --git a/src/twinkle/server/gateway/services.py b/src/twinkle/server/gateway/services.py new file mode 100644 index 00000000..b4f79f75 --- /dev/null +++ b/src/twinkle/server/gateway/services.py @@ -0,0 +1,63 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Protocol-neutral gateway use cases shared by Tinker and Twinkle adapters.""" +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from typing import Any + +from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager +from twinkle.server.lifecycle.poll_config import long_poll_window, retrieve_poll_interval + +_TERMINAL_STATUSES = frozenset({'completed', 'failed', 'cancelled'}) + + +@dataclass(frozen=True, slots=True) +class FuturePollResult: + record: dict[str, Any] | None + timed_out: bool + + +async def create_session(state: Any, metadata: dict[str, Any]) -> str: + return await state.create_session(metadata) + + +async def touch_session(state: Any, session_id: str) -> bool: + return await state.touch_session(session_id) + + +async def poll_future(state: Any, request_id: str) -> FuturePollResult: + """Long-poll one canonical future record without constructing wire responses.""" + deadline = asyncio.get_running_loop().time() + long_poll_window() + interval = retrieve_poll_interval() + record = None + while True: + record = await state.get_future(request_id) + if record is not None and record.get('status') in _TERMINAL_STATUSES: + return FuturePollResult(record=record, timed_out=False) + if asyncio.get_running_loop().time() >= deadline: + return FuturePollResult(record=record, timed_out=True) + await asyncio.sleep(interval) + + +def list_training_runs(token: str, client_type: str, *, limit: int, offset: int) -> Any: + return create_training_run_manager(token, client_type=client_type).list_runs(limit=limit, offset=offset) + + +def get_training_run(token: str, client_type: str, run_id: str, *, check_permission: bool = False) -> Any | None: + manager = create_training_run_manager(token, client_type=client_type) + if check_permission and hasattr(manager, 'get_with_permission'): + return manager.get_with_permission(run_id) + return manager.get(run_id) + + +def list_checkpoints(token: str, client_type: str, run_id: str) -> Any | None: + return create_checkpoint_manager(token, client_type=client_type).list_checkpoints(run_id) + + +def delete_checkpoint(token: str, client_type: str, run_id: str, checkpoint_id: str) -> bool: + return create_checkpoint_manager(token, client_type=client_type).delete(run_id, checkpoint_id) + + +def get_weights_info(token: str, client_type: str, path: str) -> Any | None: + return create_checkpoint_manager(token, client_type=client_type).get_weights_info(path) diff --git a/src/twinkle/server/gateway/tinker_handlers.py b/src/twinkle/server/gateway/tinker_handlers.py index 19c0f697..fd61d5e5 100644 --- a/src/twinkle/server/gateway/tinker_handlers.py +++ b/src/twinkle/server/gateway/tinker_handlers.py @@ -7,7 +7,6 @@ """ from __future__ import annotations -import asyncio from collections.abc import Callable from fastapi import Depends, FastAPI, HTTPException, Request, Response from tinker import types @@ -18,10 +17,13 @@ from twinkle.hub import HubOperation from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager -from twinkle.server.lifecycle.poll_config import long_poll_window, retrieve_poll_interval from twinkle.server.utils.auth import get_token_from_request from twinkle.server.utils.task_errors import error_payload_from_stored from twinkle.utils.logger import get_logger +from .services import create_session as create_session_use_case +from .services import delete_checkpoint +from .services import get_training_run as get_training_run_use_case +from .services import get_weights_info, list_checkpoints, list_training_runs, poll_future, touch_session logger = get_logger() @@ -57,14 +59,14 @@ async def create_session( body: types.CreateSessionRequest, self: GatewayServer = Depends(self_fn), ) -> types.CreateSessionResponse: - session_id = await self.state.create_session(body.model_dump()) + session_id = await create_session_use_case(self.state, body.model_dump()) return types.CreateSessionResponse(session_id=session_id) @app.post('/session_heartbeat') async def session_heartbeat( request: Request, body: types.SessionHeartbeatRequest, self: GatewayServer = Depends(self_fn) ) -> types.SessionHeartbeatResponse: # noqa: E125 - alive = await self.state.touch_session(body.session_id) + alive = await touch_session(self.state, body.session_id) if not alive: raise HTTPException(status_code=404, detail='Unknown session') return types.SessionHeartbeatResponse() @@ -82,35 +84,18 @@ async def retrieve_future(request: Request, self: GatewayServer = Depends(self_fn)) -> Any: """Retrieve the result of an async task with long polling.""" request_id = body.request_id - max_wait = long_poll_window() - poll_interval = retrieve_poll_interval() - start = asyncio.get_running_loop().time() - - while True: - record = await self.state.get_future(request_id) - + outcome = await poll_future(self.state, request_id) + record = outcome.record + if outcome.timed_out: + response_data: dict[str, Any] = {'type': 'try_again'} if record is not None: - status = record.get('status') - if status not in ('pending', 'queued', 'running'): - break - - # ``record is None`` here means the future hasn't been written yet - # (cross-replica visibility lag) — fold into the long-poll loop - # rather than short-circuit ``try_again``: returning immediately - # lets the SDK hammer this endpoint at ~150 Hz. - if asyncio.get_running_loop().time() - start >= max_wait: - response_data: dict[str, Any] = {'type': 'try_again'} - if record is not None: - if queue_state := record.get('queue_state'): - response_data['queue_state'] = queue_state - if queue_state_reason := record.get('queue_state_reason'): - response_data['queue_state_reason'] = queue_state_reason - return response_data - - await asyncio.sleep(poll_interval) + if queue_state := record.get('queue_state'): + response_data['queue_state'] = queue_state + if queue_state_reason := record.get('queue_state_reason'): + response_data['queue_state_reason'] = queue_state_reason + return response_data status = record.get('status') - if status == 'failed': payload = error_payload_from_stored(record.get('result'), request_id=request_id) return payload.model_dump(mode='json', exclude_none=True) @@ -118,7 +103,6 @@ async def retrieve_future(request: Request, result = record.get('result') if result is None: raise HTTPException(status_code=500, detail='Task completed but no result found') - if hasattr(result, 'model_dump'): return result.model_dump() return result @@ -128,14 +112,12 @@ async def retrieve_future(request: Request, @app.get('/training_runs') async def get_training_runs(request: Request, limit: int = 20, offset: int = 0) -> types.TrainingRunsResponse: token = get_token_from_request(request) - training_run_manager = create_training_run_manager(token, client_type='tinker') - return training_run_manager.list_runs(limit=limit, offset=offset) + return list_training_runs(token, 'tinker', limit=limit, offset=offset) @app.get('/training_runs/{run_id}') async def get_training_run(request: Request, run_id: str) -> types.TrainingRun: token = get_token_from_request(request) - training_run_manager = create_training_run_manager(token, client_type='tinker') - run = training_run_manager.get(run_id) + run = get_training_run_use_case(token, 'tinker', run_id) if not run: raise HTTPException(status_code=404, detail=f'Training run {run_id} not found') return run @@ -143,8 +125,7 @@ async def get_training_run(request: Request, run_id: str) -> types.TrainingRun: @app.get('/training_runs/{run_id}/checkpoints') async def get_run_checkpoints(request: Request, run_id: str) -> types.CheckpointsListResponse: token = get_token_from_request(request) - checkpoint_manager = create_checkpoint_manager(token, client_type='tinker') - response = checkpoint_manager.list_checkpoints(run_id) + response = list_checkpoints(token, 'tinker', run_id) if not response: raise HTTPException(status_code=404, detail=f'Training run {run_id} not found') return response @@ -152,8 +133,7 @@ async def get_run_checkpoints(request: Request, run_id: str) -> types.Checkpoint @app.delete('/training_runs/{run_id}/checkpoints/{checkpoint_id:path}') async def delete_run_checkpoint(request: Request, run_id: str, checkpoint_id: str) -> Any: token = get_token_from_request(request) - checkpoint_manager = create_checkpoint_manager(token, client_type='tinker') - success = checkpoint_manager.delete(run_id, checkpoint_id) + success = delete_checkpoint(token, 'tinker', run_id, checkpoint_id) if not success: raise HTTPException(status_code=404, detail=f'Checkpoint {checkpoint_id} not found for run {run_id}') return None @@ -161,9 +141,8 @@ async def delete_run_checkpoint(request: Request, run_id: str, checkpoint_id: st @app.post('/weights_info') async def weights_info(request: Request, body: dict[str, Any]) -> types.WeightsInfoResponse: token = get_token_from_request(request) - checkpoint_manager = create_checkpoint_manager(token, client_type='tinker') tinker_path = body.get('tinker_path') - response = checkpoint_manager.get_weights_info(tinker_path) + response = get_weights_info(token, 'tinker', tinker_path) if not response: raise HTTPException(status_code=404, detail=f'Weights at {tinker_path} not found') return response diff --git a/src/twinkle/server/gateway/twinkle_handlers.py b/src/twinkle/server/gateway/twinkle_handlers.py index b0b5366a..94c43a84 100644 --- a/src/twinkle/server/gateway/twinkle_handlers.py +++ b/src/twinkle/server/gateway/twinkle_handlers.py @@ -6,8 +6,6 @@ """ from __future__ import annotations -import asyncio -import time from collections.abc import Callable from fastapi import Depends, FastAPI, HTTPException, Request from typing import TYPE_CHECKING @@ -18,10 +16,13 @@ import twinkle_client.types as types from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager, validate_user_path from twinkle.server.lifecycle.envelope import envelope_from_record -from twinkle.server.lifecycle.poll_config import long_poll_window, retrieve_poll_interval +from twinkle.server.lifecycle.poll_config import long_poll_window from twinkle.server.utils.auth import get_token_from_request from twinkle.utils.logger import get_logger -from twinkle_client.types.lifecycle import TERMINAL_STATUSES +from .services import create_session as create_session_use_case +from .services import delete_checkpoint +from .services import get_training_run as get_training_run_use_case +from .services import get_weights_info, list_checkpoints, list_training_runs, poll_future, touch_session logger = get_logger() @@ -84,7 +85,18 @@ async def get_server_capabilities( request: Request, self: GatewayServer = Depends(self_fn), ) -> types.GetServerCapabilitiesResponse: - return types.GetServerCapabilitiesResponse(supported_models=self.supported_models) + return types.GetServerCapabilitiesResponse( + supported_models=self.supported_models, + protocol_version=1, + features=types.ClientFeatures( + task_envelope=True, + cancel=True, + data_plane=True, + full_training=True, + batch_retrieve=False, + ), + limits=types.ProtocolLimits(long_poll_timeout_seconds=long_poll_window()), + ) @app.post('/twinkle/create_session', response_model=types.CreateSessionResponse) async def create_session( @@ -92,7 +104,7 @@ async def create_session( body: types.CreateSessionRequest, self: GatewayServer = Depends(self_fn), ) -> types.CreateSessionResponse: - session_id = await self.state.create_session(body.model_dump()) + session_id = await create_session_use_case(self.state, body.model_dump()) return types.CreateSessionResponse(session_id=session_id) @app.post('/twinkle/session_heartbeat', response_model=types.SessionHeartbeatResponse) @@ -101,7 +113,7 @@ async def session_heartbeat( body: types.SessionHeartbeatRequest, self: GatewayServer = Depends(self_fn), ) -> types.SessionHeartbeatResponse: - alive = await self.state.touch_session(body.session_id) + alive = await touch_session(self.state, body.session_id) if not alive: raise HTTPException(status_code=404, detail='Unknown session') return types.SessionHeartbeatResponse() @@ -127,25 +139,10 @@ async def retrieve_future( finish. See ``poll_config`` for the numbers. """ request_id = body.request_id - deadline = time.monotonic() + long_poll_window() - interval = retrieve_poll_interval() - record = None - - while True: - record = await self.state.get_future(request_id) - if record is not None and record.get('status') in TERMINAL_STATUSES: - return envelope_from_record(request_id, record) - if time.monotonic() >= deadline: - break - await asyncio.sleep(interval) - - if record is None: - # A whole window with no record: this endpoint knows how long it - # waited, which is why the missing-record decision lives here rather - # than in the caller. Cross-replica visibility lag is folded into the - # wait loop above instead of short-circuiting to 404. + outcome = await poll_future(self.state, request_id) + if outcome.record is None: raise HTTPException(status_code=404, detail=f'request_id {request_id} not found or expired') - return envelope_from_record(request_id, record) + return envelope_from_record(request_id, outcome.record) @app.post('/twinkle/cancel', response_model=types.CancelResponse) async def cancel_future( @@ -165,14 +162,12 @@ async def cancel_future( @app.get('/twinkle/training_runs', response_model=types.TrainingRunsResponse) async def get_training_runs(request: Request, limit: int = 20, offset: int = 0) -> types.TrainingRunsResponse: token = get_token_from_request(request) - training_run_manager = create_training_run_manager(token, client_type='twinkle') - return training_run_manager.list_runs(limit=limit, offset=offset) + return list_training_runs(token, 'twinkle', limit=limit, offset=offset) @app.get('/twinkle/training_runs/{run_id}', response_model=types.TrainingRun) async def get_training_run(request: Request, run_id: str) -> types.TrainingRun: token = get_token_from_request(request) - training_run_manager = create_training_run_manager(token, client_type='twinkle') - run = training_run_manager.get_with_permission(run_id) + run = get_training_run_use_case(token, 'twinkle', run_id, check_permission=True) if not run: raise HTTPException(status_code=404, detail=f'Training run {run_id} not found or access denied') return run @@ -180,8 +175,7 @@ async def get_training_run(request: Request, run_id: str) -> types.TrainingRun: @app.get('/twinkle/training_runs/{run_id}/checkpoints', response_model=types.CheckpointsListResponse) async def get_run_checkpoints(request: Request, run_id: str) -> types.CheckpointsListResponse: token = get_token_from_request(request) - checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') - response = checkpoint_manager.list_checkpoints(run_id) + response = list_checkpoints(token, 'twinkle', run_id) if response is None: raise HTTPException(status_code=404, detail=f'Training run {run_id} not found or access denied') return response @@ -196,8 +190,7 @@ async def delete_run_checkpoint(request: Request, run_id: str, if not validate_user_path(token, checkpoint_id): raise HTTPException(status_code=400, detail='Invalid checkpoint path: path traversal not allowed') - checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') - success = checkpoint_manager.delete(run_id, checkpoint_id) + success = delete_checkpoint(token, 'twinkle', run_id, checkpoint_id) if not success: raise HTTPException(status_code=404, detail=f'Checkpoint {checkpoint_id} not found or access denied') @@ -206,8 +199,7 @@ async def delete_run_checkpoint(request: Request, run_id: str, @app.post('/twinkle/weights_info', response_model=types.WeightsInfoResponse) async def weights_info(request: Request, body: types.WeightsInfoRequest) -> types.WeightsInfoResponse: token = get_token_from_request(request) - checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') - response = checkpoint_manager.get_weights_info(body.twinkle_path) + response = get_weights_info(token, 'twinkle', body.twinkle_path) if response is None: raise HTTPException(status_code=404, detail=f'Weights at {body.twinkle_path} not found or access denied') return response diff --git a/src/twinkle_client/__init__.py b/src/twinkle_client/__init__.py index 8dc72a31..054f073c 100644 --- a/src/twinkle_client/__init__.py +++ b/src/twinkle_client/__init__.py @@ -65,7 +65,7 @@ def init_twinkle_client( An initialised :class:`~twinkle_client.manager.TwinkleClient` instance. """ from .manager import TwinkleClient - return TwinkleClient( + return TwinkleClient.connect( base_url=base_url, api_key=api_key, session_heartbeat_interval=session_heartbeat_interval, @@ -74,5 +74,6 @@ def init_twinkle_client( from .data_plane import DataPlaneClient # noqa: E402 (late import: keeps optional data-plane dep off the hot path) +from .http import ClientContext, ClientTransport # noqa: E402 -__all__ = ['DataPlaneClient', 'init_tinker_client', 'init_twinkle_client'] +__all__ = ['ClientContext', 'ClientTransport', 'DataPlaneClient', 'init_tinker_client', 'init_twinkle_client'] diff --git a/src/twinkle_client/_future.py b/src/twinkle_client/_future.py index 493b5e28..18237487 100644 --- a/src/twinkle_client/_future.py +++ b/src/twinkle_client/_future.py @@ -13,8 +13,8 @@ from typing import Any, Optional from twinkle_client.exceptions import TaskCancelledError, TaskFailedError, TaskRecordLostError, TaskWaitTimeoutError -from twinkle_client.http import http_post -from twinkle_client.http.context import get_base_url +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport from twinkle_client.types.lifecycle import TERMINAL_STATUSES, TaskEnvelope logger = logging.getLogger('twinkle_client') @@ -34,33 +34,33 @@ _TRANSPORT_RETRY_MAX = 5 -def _retrieve_url() -> str: - return f'{get_base_url()}/twinkle/retrieve_future' +def _retrieve_url(transport: ClientTransport) -> str: + return f'{transport.context.base_url}/twinkle/retrieve_future' -def _cancel_url() -> str: - return f'{get_base_url()}/twinkle/cancel' +def _cancel_url(transport: ClientTransport) -> str: + return f'{transport.context.base_url}/twinkle/cancel' -def _best_effort_cancel(request_id: str) -> None: +def _best_effort_cancel(request_id: str, transport: ClientTransport) -> None: """Ask the server to drop a task when the caller abandons the wait (e.g. Ctrl-C). Never raises: a failed cancel must not mask the original interrupt. The server only drops not-yet-started tasks, so a running task is unaffected. """ try: - http_post(url=_cancel_url(), json_data={'request_id': request_id}, timeout=2) + transport.post(_cancel_url(transport), json_data={'request_id': request_id}, timeout=2) except BaseException as e: # noqa: BLE001 - best effort; never mask the interrupt logger.debug('[future] best-effort cancel of %s failed: %s', request_id, e) -def _post_retrieve(request_id: str) -> TaskEnvelope: +def _post_retrieve(request_id: str, transport: ClientTransport) -> TaskEnvelope: """POST one retrieve and parse the reply into a TaskEnvelope. Raises ``requests.HTTPError`` (a :class:`TwinkleHTTPError` after the client error-parsing change lands) on a non-2xx response. """ - response = http_post(url=_retrieve_url(), json_data={'request_id': request_id}) + response = transport.post(_retrieve_url(transport), json_data={'request_id': request_id}) return TaskEnvelope.model_validate(response.json()) @@ -110,7 +110,13 @@ def _unwrap(env: TaskEnvelope, model_cls) -> Any: return model_cls.model_validate(env.result) if model_cls is not None else env.result -def resolve(submit: TaskEnvelope, *, model_cls, total_timeout: float = _DEFAULT_TOTAL_TIMEOUT) -> Any: +def resolve( + submit: TaskEnvelope, + *, + model_cls, + total_timeout: float = _DEFAULT_TOTAL_TIMEOUT, + transport: ClientTransport | None = None, +) -> Any: """Block until ``submit``'s task reaches a terminal state, then return its result. A terminal submit envelope is unwrapped directly, issuing no Retrieve_Endpoint @@ -124,6 +130,7 @@ def resolve(submit: TaskEnvelope, *, model_cls, total_timeout: float = _DEFAULT_ if submit.status in TERMINAL_STATUSES: return _unwrap(submit, model_cls) # same call as the retrieve path + resolved_transport = capture_transport(transport) deadline = time.monotonic() + total_timeout transport_failures = not_found_count = 0 try: @@ -131,7 +138,7 @@ def resolve(submit: TaskEnvelope, *, model_cls, total_timeout: float = _DEFAULT_ if time.monotonic() >= deadline: raise TaskWaitTimeoutError(request_id=submit.request_id, waited=total_timeout) try: - reply = _post_retrieve(submit.request_id) + reply = _post_retrieve(submit.request_id, resolved_transport) transport_failures = not_found_count = 0 except requests.HTTPError as e: status = _status_of(e) @@ -153,15 +160,14 @@ def resolve(submit: TaskEnvelope, *, model_cls, total_timeout: float = _DEFAULT_ except (KeyboardInterrupt, SystemExit): # Caller abandoned the wait: best-effort ask the server to drop the task if it # has not started, then re-raise so the interrupt is never swallowed. - _best_effort_cancel(submit.request_id) + _best_effort_cancel(submit.request_id, resolved_transport) raise -def resolve_response(response, model_cls) -> Any: - """Resolve a Submit_Endpoint HTTP response through the Client_Future_Layer. - - The one place the (already status-checked) reply's Task_Envelope is validated - and resolved, shared by every public client method so they keep synchronous - signatures without duplicating the parse+resolve step. - """ - return resolve(TaskEnvelope.model_validate(response.json()), model_cls=model_cls) +def resolve_response(response, model_cls, *, transport: ClientTransport | None = None) -> Any: + """Resolve a Submit_Endpoint response using the submitter's transport.""" + return resolve( + TaskEnvelope.model_validate(response.json()), + model_cls=model_cls, + transport=transport, + ) diff --git a/src/twinkle_client/common/component_rpc.py b/src/twinkle_client/common/component_rpc.py index 78b7d7d7..c7b631bb 100644 --- a/src/twinkle_client/common/component_rpc.py +++ b/src/twinkle_client/common/component_rpc.py @@ -1,52 +1,50 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Shared plumbing for the processor-backed component clients. - -Dataset / dataloader / processor wrappers all talk to the same two generic endpoints, -so they share one place that builds those two request bodies. Before this, each method -spread its arguments over the top level of a hand-built dict, which made the envelope -indistinguishable from its payload -- a misspelt ``processor_id`` was just another -argument. Nesting the payload under the declared passthrough region is what lets the -envelope be strict. -""" +"""Shared transport-bound plumbing for processor-backed component clients.""" from __future__ import annotations from typing import Any from twinkle_client._request_builder import build_request -from twinkle_client.http import get_base_url, http_post_model +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport from twinkle_client.types.processor import (ProcessorCallRequest, ProcessorCallResponse, ProcessorCreateRequest, ProcessorCreateResponse) -# Sentinel: "caller did not pass an HTTP timeout", so the shared default applies. A -# literal ``None`` means "no timeout at all" in ``http_post_model``, so it cannot double -# as the unset marker. _UNSET = object() -def processor_base_url() -> str: - """The single processor route prefix used by every component client.""" - return f'{get_base_url()}/processor/twinkle' +def processor_base_url(transport: ClientTransport) -> str: + return f'{transport.context.base_url}/processor/twinkle' -def create_remote_component(processor_type: str, class_type: str, **init_kwargs: Any) -> str: - """Create a server-side component and return its ``pid:``-prefixed id.""" +def create_remote_component( + processor_type: str, + class_type: str, + *, + transport: ClientTransport | None = None, + **init_kwargs: Any, +) -> str: + """Create a server-side component using one captured transport.""" + resolved = capture_transport(transport) body = build_request(ProcessorCreateRequest, processor_type=processor_type, class_type=class_type, **init_kwargs) - response = http_post_model(f'{processor_base_url()}/create', body) + response = resolved.post_model(f'{processor_base_url(resolved)}/create', body) return ProcessorCreateResponse(**response.json()).processor_id -def call_remote_component(processor_id: str, function: str, http_timeout: Any = _UNSET, /, **call_kwargs: Any) -> Any: - """Invoke ``function`` on a server-side component and return its result. - - ``http_timeout`` is positional-only so it can never be mistaken for -- or collide - with -- one of the remote callable's own arguments, which are all keywords. - - ``StopIteration`` propagates from the HTTP layer on an exhausted iterator (the - server answers 410), which is what makes a remote ``__next__`` usable in a plain - ``for`` loop. - """ +def call_remote_component( + processor_id: str, + function: str, + http_timeout: Any = _UNSET, + /, + *, + transport: ClientTransport | None = None, + **call_kwargs: Any, +) -> Any: + """Invoke one server-side component using its owner's transport.""" + resolved = capture_transport(transport) body = build_request(ProcessorCallRequest, processor_id=processor_id, function=function, **call_kwargs) - url = f'{processor_base_url()}/call' + url = f'{processor_base_url(resolved)}/call' response = ( - http_post_model(url, body) if http_timeout is _UNSET else http_post_model(url, body, timeout=http_timeout)) + resolved.post_model(url, body) if http_timeout is _UNSET else resolved.post_model( + url, body, timeout=http_timeout)) return ProcessorCallResponse(**response.json()).result diff --git a/src/twinkle_client/data_plane.py b/src/twinkle_client/data_plane.py index 18886c42..8fe9a217 100644 --- a/src/twinkle_client/data_plane.py +++ b/src/twinkle_client/data_plane.py @@ -7,7 +7,8 @@ from typing import Any, TypeVar from twinkle_client.common.json_utils import json_safe -from twinkle_client.http import get_base_url, http_post +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport from twinkle_client.types.component import DataRef, DataRowsResponse _T = TypeVar('_T') @@ -20,8 +21,9 @@ async def _call_in_thread(func: Callable[..., _T], /, *args: Any, **kwargs: Any) class DataPlaneClient: - def __init__(self, server_url: str | None = None): - self.server_url = (server_url or f'{get_base_url()}/data-plane').rstrip('/') + def __init__(self, server_url: str | None = None, *, transport: ClientTransport | None = None): + self._transport = capture_transport(transport) + self.server_url = (server_url or f'{self._transport.context.base_url}/data-plane').rstrip('/') def put( self, @@ -30,7 +32,7 @@ def put( kind: str = 'data', tags: list[dict[str, Any]] | None = None, ) -> DataRef: - response = http_post( + response = self._transport.post( f'{self.server_url}/twinkle/put', json_data={ 'rows': json_safe(rows), @@ -54,7 +56,7 @@ async def aput( return await _call_in_thread(self.put, rows, kind=kind, tags=tags) def get(self, ref: DataRef, *, fields: list[str] | None = None) -> list[dict[str, Any]]: - response = http_post( + response = self._transport.post( f'{self.server_url}/twinkle/get', json_data={ 'ref': ref.model_dump(), @@ -70,7 +72,7 @@ def get_batch( *, fields: list[str] | None = None, ) -> DataRowsResponse: - response = http_post( + response = self._transport.post( f'{self.server_url}/twinkle/get', json_data={ 'ref': ref.model_dump(), @@ -104,7 +106,7 @@ def append( *, tags: list[dict[str, Any]] | None = None, ) -> DataRef: - response = http_post( + response = self._transport.post( f'{self.server_url}/twinkle/append', json_data={ 'ref': ref.model_dump(), @@ -128,7 +130,7 @@ async def aappend( return await _call_in_thread(self.append, ref, rows, tags=tags) def release(self, ref: DataRef) -> None: - response = http_post( + response = self._transport.post( f'{self.server_url}/twinkle/release', json_data={'ref': ref.model_dump()}, ) diff --git a/src/twinkle_client/dataloader/__init__.py b/src/twinkle_client/dataloader/__init__.py index b94ed5b8..3db1e11c 100644 --- a/src/twinkle_client/dataloader/__init__.py +++ b/src/twinkle_client/dataloader/__init__.py @@ -1 +1,3 @@ from .dataloader import DataLoader + +__all__ = ['DataLoader'] diff --git a/src/twinkle_client/dataloader/dataloader.py b/src/twinkle_client/dataloader/dataloader.py index 7b0b7cf5..376b7726 100644 --- a/src/twinkle_client/dataloader/dataloader.py +++ b/src/twinkle_client/dataloader/dataloader.py @@ -4,34 +4,48 @@ from twinkle.dataset import Dataset from twinkle.processor import InputProcessor from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport class DataLoader: """Client wrapper for DataLoader that calls server HTTP endpoints.""" - def __init__(self, dataset: Union[Dataset, Callable], **kwargs): - self.processor_id = create_remote_component('dataloader', 'DataLoader', dataset=dataset, **kwargs) + def __init__( + self, + dataset: Union[Dataset, Callable], + *, + transport: ClientTransport | None = None, + **kwargs, + ): + dataset_transport = getattr(dataset, '_transport', None) + if transport is not None and dataset_transport is not None and transport is not dataset_transport: + raise ValueError('DataLoader and its remote Dataset must use the same ClientTransport') + self._transport = capture_transport(transport or dataset_transport) + self.processor_id = create_remote_component( + 'dataloader', 'DataLoader', dataset=dataset, transport=self._transport, **kwargs) + + def _call(self, function: str, *args, **kwargs): + return call_remote_component(self.processor_id, function, *args, transport=self._transport, **kwargs) def __len__(self): - return call_remote_component(self.processor_id, '__len__') + return self._call('__len__') def set_processor(self, processor_cls: Union[Type[InputProcessor], str, InputProcessor, Callable], **kwargs): - return call_remote_component(self.processor_id, 'set_processor', processor_cls=processor_cls, **kwargs) + return self._call('set_processor', processor_cls=processor_cls, **kwargs) def __iter__(self): - call_remote_component(self.processor_id, '__iter__') + self._call('__iter__') return self def __next__(self): - return call_remote_component(self.processor_id, '__next__') + return self._call('__next__') def skip_consumed_samples(self, consumed_train_samples: int): - return call_remote_component( - self.processor_id, 'skip_consumed_samples', consumed_train_samples=consumed_train_samples) + return self._call('skip_consumed_samples', consumed_train_samples=consumed_train_samples) def resume_from_checkpoint(self, consumed_train_samples, **kwargs): - return call_remote_component( - self.processor_id, 'resume_from_checkpoint', consumed_train_samples=consumed_train_samples, **kwargs) + return self._call('resume_from_checkpoint', consumed_train_samples=consumed_train_samples, **kwargs) def get_state(self): - return call_remote_component(self.processor_id, 'get_state') + return self._call('get_state') diff --git a/src/twinkle_client/dataset/__init__.py b/src/twinkle_client/dataset/__init__.py index ba37b1fe..23f7ed85 100644 --- a/src/twinkle_client/dataset/__init__.py +++ b/src/twinkle_client/dataset/__init__.py @@ -3,3 +3,5 @@ from .iterable_packing_dataset import IterablePackingDataset from .lazy_dataset import LazyDataset from .packing_dataset import PackingDataset + +__all__ = ['Dataset', 'IterableDataset', 'IterablePackingDataset', 'LazyDataset', 'PackingDataset'] diff --git a/src/twinkle_client/dataset/base.py b/src/twinkle_client/dataset/base.py index 160e521e..f22fb0d8 100644 --- a/src/twinkle_client/dataset/base.py +++ b/src/twinkle_client/dataset/base.py @@ -5,58 +5,59 @@ from twinkle.preprocessor import DataFilter, Preprocessor from twinkle.template import Template from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport class Dataset: """Client wrapper for Dataset that calls server HTTP endpoints.""" - def __init__(self, dataset_meta: DatasetMeta = None, **kwargs): - self.processor_id = create_remote_component('dataset', 'Dataset', dataset_meta=dataset_meta, **kwargs) + def __init__( + self, + dataset_meta: DatasetMeta = None, + *, + transport: ClientTransport | None = None, + **kwargs, + ): + self._transport = capture_transport(transport) + self.processor_id = create_remote_component( + 'dataset', 'Dataset', dataset_meta=dataset_meta, transport=self._transport, **kwargs) + + def _call(self, function: str, *args, **kwargs): + return call_remote_component(self.processor_id, function, *args, transport=self._transport, **kwargs) def set_template(self, template_func: Union[Template, Type[Template], str], **kwargs): - return call_remote_component(self.processor_id, 'set_template', template_func=template_func, **kwargs) + return self._call('set_template', template_func=template_func, **kwargs) def encode(self, add_generation_prompt: bool = False, timeout: Optional[int] = 600, **kwargs): - return call_remote_component( - self.processor_id, 'encode', timeout, add_generation_prompt=add_generation_prompt, **kwargs) + return self._call('encode', timeout, add_generation_prompt=add_generation_prompt, **kwargs) def check(self, **kwargs): - return call_remote_component(self.processor_id, 'check', **kwargs) + return self._call('check', **kwargs) def cast_column(self, column: str, decode: bool = True): - return call_remote_component(self.processor_id, 'cast_column', column=column, decode=decode) + return self._call('cast_column', column=column, decode=decode) def map(self, preprocess_func: Union[Preprocessor, Callable, str, Type[Preprocessor]], dataset_meta: DatasetMeta = None, init_args: Dict[str, Any] = None, **kwargs): - return call_remote_component( - self.processor_id, - 'map', - preprocess_func=preprocess_func, - dataset_meta=dataset_meta, - init_args=init_args, - **kwargs) + return self._call( + 'map', preprocess_func=preprocess_func, dataset_meta=dataset_meta, init_args=init_args, **kwargs) def filter(self, filter_func: Union[Callable, str, Type[DataFilter], DataFilter], dataset_meta: DatasetMeta = None, init_args: Dict[str, Any] = None, **kwargs): - return call_remote_component( - self.processor_id, - 'filter', - filter_func=filter_func, - dataset_meta=dataset_meta, - init_args=init_args, - **kwargs) + return self._call('filter', filter_func=filter_func, dataset_meta=dataset_meta, init_args=init_args, **kwargs) def add_dataset(self, dataset_meta: DatasetMeta, **kwargs): - return call_remote_component(self.processor_id, 'add_dataset', dataset_meta=dataset_meta, **kwargs) + return self._call('add_dataset', dataset_meta=dataset_meta, **kwargs) def mix_dataset(self, interleave=True): - return call_remote_component(self.processor_id, 'mix_dataset', interleave=interleave) + return self._call('mix_dataset', interleave=interleave) def save_as(self, output_path: str, @@ -64,20 +65,13 @@ def save_as(self, batch_size: int = 1000, mode: str = 'immediate', **kwargs): - return call_remote_component( - self.processor_id, - 'save_as', - output_path=output_path, - format=format, - batch_size=batch_size, - mode=mode, - **kwargs) + return self._call('save_as', output_path=output_path, format=format, batch_size=batch_size, mode=mode, **kwargs) def flush_save(self): - return call_remote_component(self.processor_id, 'flush_save') + return self._call('flush_save') def __getitem__(self, idx): - return call_remote_component(self.processor_id, '__getitem__', idx=idx) + return self._call('__getitem__', idx=idx) def __len__(self): - return call_remote_component(self.processor_id, '__len__') + return self._call('__len__') diff --git a/src/twinkle_client/dataset/iterable_dataset.py b/src/twinkle_client/dataset/iterable_dataset.py index 0d7b5665..a7ad36e9 100644 --- a/src/twinkle_client/dataset/iterable_dataset.py +++ b/src/twinkle_client/dataset/iterable_dataset.py @@ -3,26 +3,39 @@ from twinkle.dataset import DatasetMeta from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport class IterableDataset(TorchIterableDataset): """Client wrapper for IterableDataset that calls server HTTP endpoints.""" - def __init__(self, dataset_meta: DatasetMeta = None, **kwargs): - self.processor_id = create_remote_component('dataset', 'IterableDataset', dataset_meta=dataset_meta, **kwargs) + def __init__( + self, + dataset_meta: DatasetMeta = None, + *, + transport: ClientTransport | None = None, + **kwargs, + ): + self._transport = capture_transport(transport) + self.processor_id = create_remote_component( + 'dataset', 'IterableDataset', dataset_meta=dataset_meta, transport=self._transport, **kwargs) + + def _call(self, function: str, *args, **kwargs): + return call_remote_component(self.processor_id, function, *args, transport=self._transport, **kwargs) def add_dataset(self, dataset_meta: DatasetMeta, **kwargs): - return call_remote_component(self.processor_id, 'add_dataset', dataset_meta=dataset_meta, **kwargs) + return self._call('add_dataset', dataset_meta=dataset_meta, **kwargs) def __len__(self): - return call_remote_component(self.processor_id, '__len__') + return self._call('__len__') def __getitem__(self, idx): - return call_remote_component(self.processor_id, '__getitem__', idx=idx) + return self._call('__getitem__', idx=idx) def __iter__(self): - call_remote_component(self.processor_id, '__iter__') + self._call('__iter__') return self def __next__(self): - return call_remote_component(self.processor_id, '__next__') + return self._call('__next__') diff --git a/src/twinkle_client/dataset/iterable_packing_dataset.py b/src/twinkle_client/dataset/iterable_packing_dataset.py index 012b5643..42991a6d 100644 --- a/src/twinkle_client/dataset/iterable_packing_dataset.py +++ b/src/twinkle_client/dataset/iterable_packing_dataset.py @@ -5,17 +5,24 @@ from twinkle.dataset import DatasetMeta from twinkle.template import Template from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport class IterablePackingDataset(IterableDataset): """Client wrapper for IterablePackingDataset that calls server HTTP endpoints.""" - def __init__(self, - dataset_meta: DatasetMeta = None, - packing_interval: int = 128, - packing_num_proc: int = 1, - cyclic: bool = False, - **kwargs): + def __init__( + self, + dataset_meta: DatasetMeta = None, + packing_interval: int = 128, + packing_num_proc: int = 1, + cyclic: bool = False, + *, + transport: ClientTransport | None = None, + **kwargs, + ): + self._transport = capture_transport(transport) self.processor_id = create_remote_component( 'dataset', 'IterablePackingDataset', @@ -23,17 +30,22 @@ def __init__(self, packing_interval=packing_interval, packing_num_proc=packing_num_proc, cyclic=cyclic, - **kwargs) + transport=self._transport, + **kwargs, + ) + + def _call(self, function: str, *args, **kwargs): + return call_remote_component(self.processor_id, function, *args, transport=self._transport, **kwargs) def set_template(self, template_cls: Union[Type[Template], str, Template], **kwargs): - return call_remote_component(self.processor_id, 'set_template', template_cls=template_cls, **kwargs) + return self._call('set_template', template_cls=template_cls, **kwargs) def pack_dataset(self): - return call_remote_component(self.processor_id, 'pack_dataset') + return self._call('pack_dataset') def __iter__(self): - call_remote_component(self.processor_id, '__iter__') + self._call('__iter__') return self def __next__(self): - return call_remote_component(self.processor_id, '__next__') + return self._call('__next__') diff --git a/src/twinkle_client/dataset/lazy_dataset.py b/src/twinkle_client/dataset/lazy_dataset.py index b1af790a..18f95857 100644 --- a/src/twinkle_client/dataset/lazy_dataset.py +++ b/src/twinkle_client/dataset/lazy_dataset.py @@ -3,57 +3,55 @@ from twinkle.dataset import DatasetMeta from twinkle.preprocessor import DataFilter, Preprocessor -from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.common.component_rpc import create_remote_component +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport from .base import Dataset class LazyDataset(Dataset): """Client wrapper for LazyDataset that calls server HTTP endpoints.""" - def __init__(self, dataset_meta: DatasetMeta = None, **kwargs): - self.processor_id = create_remote_component('dataset', 'LazyDataset', dataset_meta=dataset_meta, **kwargs) + def __init__( + self, + dataset_meta: DatasetMeta = None, + *, + transport: ClientTransport | None = None, + **kwargs, + ): + self._transport = capture_transport(transport) + self.processor_id = create_remote_component( + 'dataset', 'LazyDataset', dataset_meta=dataset_meta, transport=self._transport, **kwargs) def map(self, preprocess_func: Union[Preprocessor, Callable, str, Type[Preprocessor]], dataset_meta: DatasetMeta = None, init_args: Dict[str, Any] = None, **kwargs): - return call_remote_component( - self.processor_id, - 'map', - preprocess_func=preprocess_func, - dataset_meta=dataset_meta, - init_args=init_args, - **kwargs) + return self._call( + 'map', preprocess_func=preprocess_func, dataset_meta=dataset_meta, init_args=init_args, **kwargs) def filter(self, filter_func: Union[Callable, str, Type[DataFilter], DataFilter], dataset_meta: DatasetMeta = None, init_args: Dict[str, Any] = None, **kwargs): - return call_remote_component( - self.processor_id, - 'filter', - filter_func=filter_func, - dataset_meta=dataset_meta, - init_args=init_args, - **kwargs) + return self._call('filter', filter_func=filter_func, dataset_meta=dataset_meta, init_args=init_args, **kwargs) def add_dataset(self, dataset_meta: DatasetMeta, **kwargs): - return call_remote_component(self.processor_id, 'add_dataset', dataset_meta=dataset_meta, **kwargs) + return self._call('add_dataset', dataset_meta=dataset_meta, **kwargs) def mix_dataset(self, interleave=True): - return call_remote_component(self.processor_id, 'mix_dataset', interleave=interleave) + return self._call('mix_dataset', interleave=interleave) def encode(self, add_generation_prompt: bool = False, timeout: Optional[int] = 600, **kwargs): - return call_remote_component( - self.processor_id, 'encode', timeout, add_generation_prompt=add_generation_prompt, **kwargs) + return self._call('encode', timeout, add_generation_prompt=add_generation_prompt, **kwargs) def check(self, **kwargs): - return call_remote_component(self.processor_id, 'check', **kwargs) + return self._call('check', **kwargs) def __getitem__(self, idx): - return call_remote_component(self.processor_id, '__getitem__', idx=idx) + return self._call('__getitem__', idx=idx) def __len__(self): - return call_remote_component(self.processor_id, '__len__') + return self._call('__len__') diff --git a/src/twinkle_client/dataset/packing_dataset.py b/src/twinkle_client/dataset/packing_dataset.py index cbf03d48..b17185e0 100644 --- a/src/twinkle_client/dataset/packing_dataset.py +++ b/src/twinkle_client/dataset/packing_dataset.py @@ -1,21 +1,37 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from twinkle.dataset import DatasetMeta -from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.common.component_rpc import create_remote_component +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport from .base import Dataset class PackingDataset(Dataset): """Client wrapper for PackingDataset that calls server HTTP endpoints.""" - def __init__(self, dataset_meta: DatasetMeta = None, packing_num_proc: int = 1, **kwargs): + def __init__( + self, + dataset_meta: DatasetMeta = None, + packing_num_proc: int = 1, + *, + transport: ClientTransport | None = None, + **kwargs, + ): + self._transport = capture_transport(transport) self.processor_id = create_remote_component( - 'dataset', 'PackingDataset', dataset_meta=dataset_meta, packing_num_proc=packing_num_proc, **kwargs) + 'dataset', + 'PackingDataset', + dataset_meta=dataset_meta, + packing_num_proc=packing_num_proc, + transport=self._transport, + **kwargs, + ) def pack_dataset(self): - return call_remote_component(self.processor_id, 'pack_dataset') + return self._call('pack_dataset') def __getitem__(self, index): - return call_remote_component(self.processor_id, '__getitem__', index=index) + return self._call('__getitem__', index=index) def __len__(self): - return call_remote_component(self.processor_id, '__len__') + return self._call('__len__') diff --git a/src/twinkle_client/http/__init__.py b/src/twinkle_client/http/__init__.py index ab6f48e9..55d93e97 100644 --- a/src/twinkle_client/http/__init__.py +++ b/src/twinkle_client/http/__init__.py @@ -1,20 +1,12 @@ -from .client import http_delete, http_get, http_post, http_post_model -from .context import (TWINKLE_SERVER_TOKEN, TWINKLE_SERVER_URL, get_api_key, get_base_url, get_request_id, - get_session_id, set_api_key, set_base_url, set_request_id, set_session_id) +"""Public HTTP transport API.""" +from .client import ClientTransport, http_delete, http_get, http_post, http_post_model +from .context import ClientContext __all__ = [ + 'ClientContext', + 'ClientTransport', 'http_get', 'http_post', 'http_post_model', 'http_delete', - 'TWINKLE_SERVER_URL', - 'TWINKLE_SERVER_TOKEN', - 'set_base_url', - 'get_base_url', - 'set_api_key', - 'get_api_key', - 'set_session_id', - 'get_session_id', - 'set_request_id', - 'get_request_id', ] diff --git a/src/twinkle_client/http/client.py b/src/twinkle_client/http/client.py index 6ec5488c..5f7feab4 100644 --- a/src/twinkle_client/http/client.py +++ b/src/twinkle_client/http/client.py @@ -1,218 +1,259 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Instance-owned HTTP transport and legacy module-level compatibility facade.""" +from __future__ import annotations + import requests -from typing import Any, Dict, Optional +from collections.abc import Mapping +from dataclasses import asdict, is_dataclass +from pydantic import BaseModel +from typing import Any -from twinkle_client.exceptions import TwinkleHTTPError -from .context import get_api_key, get_base_url, get_request_id, get_session_id +from twinkle_client.exceptions import TwinkleClientValidationError, TwinkleHTTPError +from .context import ClientContext, capture_transport from .headers import build_routing_headers -# Single shared HTTP timeout for every client request (was three separate 600s). -# Must be <= 120 and strictly greater than the server Long_Poll_Window (default 30), -# so a retrieve that waits a full window still completes within the timeout and, being -# < a typical 60s gateway idle limit, survives the gateway. +# Must be greater than the server long-poll window and below common gateway idle limits. _HTTP_TIMEOUT = 90 - - -def _build_headers(additional_headers: Optional[Dict[str, str]] = None) -> Dict[str, str]: - """ - Build HTTP headers with request ID and authorization. - - Args: - additional_headers: Additional headers to include - - Returns: - Dictionary of headers - """ - headers = build_routing_headers(get_request_id(), 'Bearer ' + get_api_key()) - - if session_id := get_session_id(): - headers['X-Twinkle-Session-Id'] = session_id - - if additional_headers: - headers.update(additional_headers) - - return headers - - -def _serialize_params(params: Dict[str, Any]) -> Dict[str, Any]: - """ - Serialize parameters, handling special objects like processors. - - Args: - params: Parameters to serialize - - Returns: - Serialized parameters dictionary - """ - serialized = {} - for key, value in params.items(): - if hasattr(value, 'processor_id'): - serialized[key] = value.processor_id - elif hasattr(value, '__dict__'): - from twinkle_client.common.serialize import serialize_object - serialized[key] = serialize_object(value) - else: - serialized[key] = value - return serialized +_UNSET = object() +_JSON_PRIMITIVES = (str, int, float, bool, type(None)) + + +def _serialize_value(value: Any) -> Any: + if isinstance(value, _JSON_PRIMITIVES): + return value + if isinstance(value, bytes | bytearray | memoryview): + raise TwinkleClientValidationError('Binary values are not supported by the JSON transport') + if isinstance(value, BaseModel): + return value.model_dump(mode='json') + if is_dataclass(value) and not isinstance(value, type): + return _serialize_value(asdict(value)) + if isinstance(value, Mapping): + return {str(key): _serialize_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_serialize_value(item) for item in value] + processor_id = getattr(value, 'processor_id', None) + if isinstance(processor_id, str): + return processor_id + raise TwinkleClientValidationError( + f'Unsupported wire value {type(value).__name__}; use a Pydantic model, dataclass, or JSON value') def _handle_response(response: requests.Response) -> requests.Response: - """Handle common response processing. - - Raises: - StopIteration: When server returns HTTP 410 (iterator exhausted). - TwinkleHTTPError: When the server returns a 4xx/5xx (other than 410). It - inherits ``requests.HTTPError`` so existing ``except`` clauses keep - working, and carries the server's top-level ``error_code`` / ``category`` - / ``request_id`` when present. When those fields are absent (FastAPI's - built-in 404/405, or a gateway passthrough), it falls back to ``detail`` - with ``category='Unknown'``. - """ - # Convert HTTP 410 Gone to StopIteration (an iterator has been exhausted). if response.status_code == 410: - raise StopIteration(response.json().get('detail', 'Iterator exhausted')) - - if not response.ok: try: - body = response.json() + detail = response.json().get('detail', 'Iterator exhausted') except Exception: - body = None - if isinstance(body, dict): - category = body.get('category', 'Unknown') - error_code = body.get('error_code') - request_id = body.get('request_id') - summary = body.get('error') or body.get('detail') or response.text - else: - category, error_code, request_id, summary = 'Unknown', None, None, response.text - http_error_msg = (f'{response.status_code} Error for url: {response.url}\n' - f'Server detail:\n{summary}') - raise TwinkleHTTPError( - http_error_msg, - response=response, - status_code=response.status_code, - error_code=error_code, - category=category, - request_id=request_id, + detail = response.text or 'Iterator exhausted' + raise StopIteration(detail) + + if response.ok: + return response + + try: + body = response.json() + except Exception: + body = None + if isinstance(body, dict): + category = body.get('category', 'Unknown') + error_code = body.get('error_code') + request_id = body.get('request_id') + summary = body.get('error') or body.get('detail') or response.text + else: + category, error_code, request_id, summary = 'Unknown', None, None, response.text + message = f'{response.status_code} Error for url: {response.url}\nServer detail:\n{summary}' + raise TwinkleHTTPError( + message, + response=response, + status_code=response.status_code, + error_code=error_code, + category=category, + request_id=request_id, + ) + + +class ClientTransport: + """The sole request-time owner of URL, identity, headers, and HTTP resources.""" + + def __init__( + self, + context: ClientContext, + *, + session: requests.Session | None = None, + timeout: float = _HTTP_TIMEOUT, + ) -> None: + self._context = context + self._session = session or requests.Session() + self._timeout = timeout + self._closed = False + self._capabilities: object | None = None + + @property + def context(self) -> ClientContext: + return self._context + + @property + def closed(self) -> bool: + return self._closed + + def bind_context(self, context: ClientContext) -> None: + """Replace provisional identity before the transport is published to wrappers.""" + self._ensure_open() + self._context = context + + @property + def cached_capabilities(self) -> object | None: + return self._capabilities + + @cached_capabilities.setter + def cached_capabilities(self, value: object) -> None: + self._capabilities = value + + def url(self, path_or_url: str = '') -> str: + if path_or_url.startswith(('http://', 'https://')): + return path_or_url + if not path_or_url: + return self._context.base_url + return f'{self._context.base_url}/{path_or_url.lstrip("/")}' + + def _headers(self, additional_headers: Mapping[str, str] | None = None) -> dict[str, str]: + headers = build_routing_headers(self._context.routing_id, f'Bearer {self._context.api_key}') + if self._context.session_id: + headers['X-Twinkle-Session-Id'] = self._context.session_id + if additional_headers: + headers.update(additional_headers) + return headers + + def _request_timeout(self, timeout: object) -> float | None: + return self._timeout if timeout is _UNSET else timeout # type: ignore[return-value] + + def _ensure_open(self) -> None: + if self._closed: + raise RuntimeError('ClientTransport is closed') + + def get( + self, + path_or_url: str = '', + *, + params: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + timeout: float | None | object = _UNSET, + ) -> requests.Response: + self._ensure_open() + response = self._session.get( + self.url(path_or_url), + headers=self._headers(headers), + params=_serialize_value(params or {}), + timeout=self._request_timeout(timeout), + ) + return _handle_response(response) + + def post( + self, + path_or_url: str = '', + *, + json_data: Mapping[str, Any] | None = None, + data: Any = None, + headers: Mapping[str, str] | None = None, + timeout: float | None | object = _UNSET, + ) -> requests.Response: + self._ensure_open() + if isinstance(data, (bytes, bytearray, memoryview)): + raise TwinkleClientValidationError('Binary request bodies are not supported by this transport') + response = self._session.post( + self.url(path_or_url), + headers=self._headers(headers), + json=_serialize_value(json_data or {}), + data=data, + timeout=self._request_timeout(timeout), ) + return _handle_response(response) + + def post_model( + self, + path_or_url: str, + body: Any, + *, + headers: Mapping[str, str] | None = None, + timeout: float | None | object = _UNSET, + ) -> requests.Response: + from twinkle_client._request_builder import request_json + self._ensure_open() + request_headers = {'content-type': 'application/json', **dict(headers or {})} + response = self._session.post( + self.url(path_or_url), + headers=self._headers(request_headers), + data=request_json(body), + timeout=self._request_timeout(timeout), + ) + return _handle_response(response) + + def delete( + self, + path_or_url: str = '', + *, + params: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + timeout: float | None | object = _UNSET, + ) -> requests.Response: + self._ensure_open() + response = self._session.delete( + self.url(path_or_url), + headers=self._headers(headers), + params=_serialize_value(params or {}), + timeout=self._request_timeout(timeout), + ) + return _handle_response(response) - return response + def close(self) -> None: + if self._closed: + return + self._closed = True + self._session.close() +# Compatibility facade. Core wrappers always pass their captured transport explicitly; +# only legacy external callers may omit it and resolve the current default here. def http_get( - url: Optional[str] = None, - params: Optional[Dict[str, Any]] = {}, - additional_headers: Optional[Dict[str, str]] = {}, - timeout: int = _HTTP_TIMEOUT, + url: str | None = None, + params: Mapping[str, Any] | None = None, + additional_headers: Mapping[str, str] | None = None, + timeout: float | None = _HTTP_TIMEOUT, + *, + transport: ClientTransport | None = None, ) -> requests.Response: - """ - Send HTTP GET request with required headers. - - Args: - url: The target URL - params: Query parameters - additional_headers: Additional headers to include - timeout: Request timeout in seconds - - Returns: - requests.Response object - """ - url = url or get_base_url() - headers = _build_headers(additional_headers) - serialized_params = _serialize_params(params) - - response = requests.get( - url, - headers=headers, - params=serialized_params, - timeout=timeout, - ) - - return _handle_response(response) + return capture_transport(transport).get(url or '', params=params, headers=additional_headers, timeout=timeout) def http_post( - url: Optional[str] = None, - json_data: Optional[Dict[str, Any]] = {}, - data: Optional[Any] = {}, - additional_headers: Optional[Dict[str, str]] = {}, - timeout: Optional[int] = _HTTP_TIMEOUT, + url: str | None = None, + json_data: Mapping[str, Any] | None = None, + data: Any = None, + additional_headers: Mapping[str, str] | None = None, + timeout: float | None = _HTTP_TIMEOUT, + *, + transport: ClientTransport | None = None, ) -> requests.Response: - """ - Send HTTP POST request with required headers. - - Args: - url: The target URL - json_data: JSON data to send in request body - data: Form data or raw data to send in request body - additional_headers: Additional headers to include - timeout: Request timeout in seconds; None disables the timeout. - - Returns: - requests.Response object - - Raises: - StopIteration: When server returns HTTP 410 (iterator exhausted) - """ - url = url or get_base_url() - headers = _build_headers(additional_headers) - serialized_json = _serialize_params(json_data) - - response = requests.post( - url, - headers=headers, - json=serialized_json, - data=data, - timeout=timeout, - ) - - return _handle_response(response) + return capture_transport(transport).post( + url or '', json_data=json_data, data=data, headers=additional_headers, timeout=timeout) def http_post_model( url: str, body: Any, - additional_headers: Optional[Dict[str, str]] = None, - timeout: Optional[int] = _HTTP_TIMEOUT, + additional_headers: Mapping[str, str] | None = None, + timeout: float | None = _HTTP_TIMEOUT, + *, + transport: ClientTransport | None = None, ) -> requests.Response: - """POST a request model as its own JSON serialization. - - One pydantic-core pass produces the bytes, instead of ``model_dump`` followed by - ``_serialize_params``' Python-level walk and then ``json.dumps`` -- three traversals - of a body whose ``inputs`` can hold millions of elements. The model has already - validated everything, so nothing here needs to inspect the payload again. - """ - from twinkle_client._request_builder import request_json - headers = _build_headers({'content-type': 'application/json', **(additional_headers or {})}) - response = requests.post(url, headers=headers, data=request_json(body), timeout=timeout) - return _handle_response(response) + return capture_transport(transport).post_model(url, body, headers=additional_headers, timeout=timeout) def http_delete( - url: Optional[str] = None, - params: Optional[Dict[str, Any]] = {}, - additional_headers: Optional[Dict[str, str]] = {}, - timeout: int = _HTTP_TIMEOUT, + url: str | None = None, + params: Mapping[str, Any] | None = None, + additional_headers: Mapping[str, str] | None = None, + timeout: float | None = _HTTP_TIMEOUT, + *, + transport: ClientTransport | None = None, ) -> requests.Response: - """ - Send HTTP DELETE request with required headers. - - Args: - url: The target URL - params: Query parameters - additional_headers: Additional headers to include - timeout: Request timeout in seconds - - Returns: - requests.Response object - """ - url = url or get_base_url() - headers = _build_headers(additional_headers) - serialized_params = _serialize_params(params) - - response = requests.delete( - url, - headers=headers, - params=serialized_params, - timeout=timeout, - ) - - return _handle_response(response) + return capture_transport(transport).delete(url or '', params=params, headers=additional_headers, timeout=timeout) diff --git a/src/twinkle_client/http/context.py b/src/twinkle_client/http/context.py index f5b34835..858fa978 100644 --- a/src/twinkle_client/http/context.py +++ b/src/twinkle_client/http/context.py @@ -1,64 +1,86 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Immutable client identity and the compatibility default-transport registry.""" +from __future__ import annotations + import os +import threading import uuid -from datetime import datetime -from typing import Optional +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .client import ClientTransport TWINKLE_SERVER_URL = os.environ.get('TWINKLE_SERVER_URL', 'http://127.0.0.1:8000') TWINKLE_SERVER_TOKEN = os.environ.get('TWINKLE_SERVER_TOKEN', 'EMPTY_TOKEN') -# Global variables for configuration -_base_url: Optional[str] = None -_api_key: Optional[str] = None -_session_id: Optional[str] = None -_request_id: Optional[str] = None +def _normalize_base_url(base_url: str) -> str: + base_url = base_url.rstrip('/') + return base_url if base_url.endswith('/api/v1') else f'{base_url}/api/v1' -def set_base_url(url: str): - """Set the base URL for HTTP requests.""" - global _base_url - _base_url = url.rstrip('/') +@dataclass(frozen=True, slots=True) +class ClientContext: + """A resolved request identity captured by one transport.""" -def get_base_url() -> str: - """Get the current base URL.""" - base_url = _base_url or TWINKLE_SERVER_URL - if not base_url.endswith('/api/v1'): - base_url += '/api/v1' - return base_url + base_url: str + api_key: str + session_id: str | None = None + routing_id: str = '' + def __post_init__(self) -> None: + object.__setattr__(self, 'base_url', _normalize_base_url(self.base_url)) + if not self.routing_id: + object.__setattr__(self, 'routing_id', uuid.uuid4().hex) -def set_api_key(api_key: str): - """Set the API key for HTTP requests.""" - global _api_key - _api_key = api_key +_default_lock = threading.RLock() +_default_transport: ClientTransport | None = None -def get_api_key() -> str: - """Get the current API key.""" - return _api_key or TWINKLE_SERVER_TOKEN + +def _new_env_transport() -> ClientTransport: + from .client import ClientTransport + return ClientTransport( + ClientContext( + base_url=os.environ.get('TWINKLE_SERVER_URL', TWINKLE_SERVER_URL), + api_key=os.environ.get('TWINKLE_SERVER_TOKEN', TWINKLE_SERVER_TOKEN), + )) -def set_session_id(session_id: str): - """Set the session ID.""" - global _session_id - _session_id = session_id +def capture_transport(explicit: ClientTransport | None = None) -> ClientTransport: + """Return an explicit transport or capture the current compatibility default.""" + global _default_transport + if explicit is not None: + if explicit.closed: + raise RuntimeError('Cannot capture a closed ClientTransport') + return explicit + with _default_lock: + if _default_transport is None or _default_transport.closed: + _default_transport = _new_env_transport() + return _default_transport -def get_session_id() -> Optional[str]: - """Get the current session ID.""" - return _session_id +def set_default_transport(transport: ClientTransport) -> None: + if transport.closed: + raise RuntimeError('Cannot register a closed ClientTransport') + global _default_transport + with _default_lock: + _default_transport = transport -def set_request_id(request_id: str): - """Set the global request ID for HTTP requests (shared across all threads).""" - global _request_id - _request_id = request_id +def clear_default_transport(transport: ClientTransport) -> None: + global _default_transport + with _default_lock: + if _default_transport is transport: + _default_transport = None + + +# Private compatibility seam for the deferred Tinker monkey patch. New Twinkle code +# must use ClientTransport directly; these names are intentionally not re-exported. +def get_api_key() -> str: + return capture_transport().context.api_key def get_request_id() -> str: - """Get the global request ID or generate and cache a new one.""" - global _request_id - if _request_id is not None: - return _request_id - _request_id = datetime.now().strftime('%Y%m%d_%H%M%S') + '-' + str(uuid.uuid4().hex)[0:8] - return _request_id + return capture_transport().context.routing_id diff --git a/src/twinkle_client/manager.py b/src/twinkle_client/manager.py index f08e2a16..92effd5b 100644 --- a/src/twinkle_client/manager.py +++ b/src/twinkle_client/manager.py @@ -2,81 +2,105 @@ from __future__ import annotations import atexit +import os import threading -from typing import Any, Dict, List, Optional, Tuple +from dataclasses import replace +from typing import Any from twinkle import get_logger +from twinkle_client.exceptions import TwinkleHTTPError +from twinkle_client.http import ClientContext, ClientTransport +from twinkle_client.http.context import (TWINKLE_SERVER_TOKEN, TWINKLE_SERVER_URL, clear_default_transport, + set_default_transport) from twinkle_client.types.server import CapacityInfoResponse, DeleteCheckpointResponse, GetServerCapabilitiesResponse -from twinkle_client.types.session import (CreateSessionRequest, CreateSessionResponse, SessionHeartbeatRequest, - SessionHeartbeatResponse) +from twinkle_client.types.session import CreateSessionRequest, CreateSessionResponse, SessionHeartbeatRequest from twinkle_client.types.training import (Checkpoint, Cursor, ParsedCheckpointTwinklePath, TrainingRun, - TrainingRunsResponse, WeightsInfoResponse) -from .http import get_api_key, get_base_url, http_delete, http_get, http_post, set_api_key, set_base_url, set_session_id + WeightsInfoResponse) logger = get_logger() - -class TwinkleClientError(Exception): - """Base exception for TwinkleManager errors.""" - pass +# Compatibility import: HTTP failures now have one canonical implementation. +TwinkleClientError = TwinkleHTTPError class TwinkleClient: - """ - Client manager for interacting with Twinkle REST API. - - On initialization this client: - - Sets the base_url and api_key into the shared context so that all other - client objects (MultiLoraTransformersModel, vLLMSampler, processor clients) - automatically pick up the same configuration. - - Creates a server-side session and stores the session_id in context so that - every outgoing HTTP request carries it in the ``X-Twinkle-Session-Id`` header. - - Starts a lightweight background thread that touches the session every - ``session_heartbeat_interval`` seconds to keep it alive. - - Args: - base_url: Base URL of the Twinkle server (e.g. "http://localhost:8000"). - Falls back to the ``TWINKLE_SERVER_URL`` environment variable. - api_key: API key for authentication. Falls back to the - ``TWINKLE_SERVER_TOKEN`` environment variable. - route_prefix: API route prefix (default: "/twinkle"). - session_heartbeat_interval: Seconds between session touch calls (default: 30). - session_metadata: Optional metadata dict stored with the session on the server. + """Owner of one connected transport, remote session, and heartbeat thread. + + Use :meth:`connect` (normally through ``init_twinkle_client``) for remote I/O. + ``__init__`` only accepts already-established state, which keeps partial + connection failures from publishing a client or leaking a heartbeat thread. """ def __init__( self, + *, + transport: ClientTransport, + route_prefix: str = '/twinkle', + session_heartbeat_interval: int = 10, + ) -> None: + """Build an already-connected client without performing remote I/O.""" + self._transport = transport + self.base_url = transport.context.base_url + self.api_key = transport.context.api_key + self.route_prefix = route_prefix.rstrip('/') if route_prefix else '' + self._session_id = transport.context.session_id + self._heartbeat_interval = session_heartbeat_interval + self._stop_event = threading.Event() + self._heartbeat_thread: threading.Thread | None = None + self._close_lock = threading.Lock() + self._closed = False + + @classmethod + def connect( + cls, base_url: str | None = None, api_key: str | None = None, route_prefix: str | None = '/twinkle', session_heartbeat_interval: int = 10, session_metadata: dict[str, Any] | None = None, - ): - # Resolve and store config, then propagate to context so all generated - # client objects that call get_base_url() / get_api_key() get these values. - if base_url: - set_base_url(base_url) - if api_key: - set_api_key(api_key) - - self.base_url = get_base_url() - self.api_key = get_api_key() - self.route_prefix = route_prefix.rstrip('/') if route_prefix else '' - - # Create a server-side session. - self._session_id: str = self.create_session(session_metadata) - set_session_id(self._session_id) - - # Start background session-touch thread. - self._heartbeat_interval = session_heartbeat_interval - self._stop_event = threading.Event() + ) -> TwinkleClient: + """Create the remote session and atomically publish a connected client.""" + context = ClientContext( + base_url=base_url or os.environ.get('TWINKLE_SERVER_URL', TWINKLE_SERVER_URL), + api_key=api_key or os.environ.get('TWINKLE_SERVER_TOKEN', TWINKLE_SERVER_TOKEN), + ) + transport = ClientTransport(context) + prefix = route_prefix.rstrip('/') if route_prefix else '' + client = None + try: + response = transport.post( + f'{context.base_url}{prefix}/create_session', + json_data=CreateSessionRequest(metadata=session_metadata).model_dump(), + ) + session_id = CreateSessionResponse.model_validate(response.json()).session_id + transport.bind_context(replace(context, session_id=session_id)) + client = cls( + transport=transport, + route_prefix=prefix, + session_heartbeat_interval=session_heartbeat_interval, + ) + set_default_transport(transport) + client._start_heartbeat() + atexit.register(client.close) + return client + except BaseException: + if client is None: + transport.close() + else: + client.close() + raise + + @property + def transport(self) -> ClientTransport: + return self._transport + + def _start_heartbeat(self) -> None: self._heartbeat_thread = threading.Thread( target=self._touch_session_loop, daemon=True, name='TwinkleSessionHeartbeat', ) self._heartbeat_thread.start() - atexit.register(self.close) def get_capacity_info(self) -> CapacityInfoResponse: """ @@ -89,8 +113,8 @@ def get_capacity_info(self) -> CapacityInfoResponse: Raises: TwinkleClientError: If the request fails. """ - response = http_get(self._get_url('/capacity_info')) - data = self._handle_response(response) + response = self._transport.get(self._get_url('/capacity_info')) + data = response.json() return CapacityInfoResponse(**data) # ------------------------------------------------------------------ @@ -101,17 +125,6 @@ def _get_url(self, endpoint: str) -> str: """Construct full URL for an endpoint.""" return f'{self.base_url}{self.route_prefix}{endpoint}' - def _handle_response(self, response, expected_code: int = 200) -> dict[str, Any]: - """Handle HTTP response and raise appropriate errors.""" - if response.status_code != expected_code: - try: - error_data = response.json() - detail = error_data.get('detail', str(error_data)) - except Exception: - detail = response.text - raise TwinkleClientError(f'Request failed with status {response.status_code}: {detail}') - return response.json() - def create_session(self, metadata: dict[str, Any] | None = None) -> str: """ Create a server-side session. @@ -125,11 +138,10 @@ def create_session(self, metadata: dict[str, Any] | None = None) -> str: Raises: TwinkleClientError: If the session creation request fails. """ - resp = http_post( + resp = self._transport.post( self._get_url('/create_session'), json_data=CreateSessionRequest(metadata=metadata).model_dump(), ) - resp.raise_for_status() return CreateSessionResponse(**resp.json()).session_id def _touch_session_loop(self) -> None: @@ -146,12 +158,11 @@ def _touch_session_loop(self) -> None: success = False try: logger.debug(f'[TwinkleClient] Touching session (session={self._session_id})...') - resp = http_post( + self._transport.post( self._get_url('/session_heartbeat'), json_data=SessionHeartbeatRequest(session_id=self._session_id).model_dump(), timeout=min(self._heartbeat_interval, 10), ) - resp.raise_for_status() success = True except Exception as e: logger.error(f'[TwinkleClient] Session heartbeat error: {e}') @@ -162,10 +173,36 @@ def _touch_session_loop(self) -> None: self._stop_event.wait(timeout=sleep_time) def close(self) -> None: - """Stop the background heartbeat thread and clear session context.""" + """Stop owned resources exactly once without affecting another client.""" + with self._close_lock: + if self._closed: + return + self._closed = True self._stop_event.set() - if self._heartbeat_thread.is_alive(): - self._heartbeat_thread.join(timeout=2) + if self._heartbeat_thread is not None and self._heartbeat_thread.is_alive(): + self._heartbeat_thread.join(timeout=max(2, min(self._heartbeat_interval, 10))) + clear_default_transport(self._transport) + self._transport.close() + try: + atexit.unregister(self.close) + except Exception: + pass + + def __enter__(self) -> TwinkleClient: + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.close() + + def model(self, model_id: str, **kwargs: Any): + """Create a remote training model bound explicitly to this client.""" + from twinkle_client.model import MultiLoraTransformersModel + return MultiLoraTransformersModel(model_id, transport=self._transport, **kwargs) + + def sampler(self, model_id: str, **kwargs: Any): + """Create a remote sampler bound explicitly to this client.""" + from twinkle_client.sampler import vLLMSampler + return vLLMSampler(model_id, transport=self._transport, **kwargs) # ------------------------------------------------------------------ # Health Check @@ -179,7 +216,7 @@ def health_check(self) -> bool: True if server is healthy, False otherwise. """ try: - response = http_get(self._get_url('/healthz')) + response = self._transport.get(self._get_url('/healthz')) return response.status_code == 200 except Exception: return False @@ -195,9 +232,13 @@ def get_server_capabilities(self) -> GetServerCapabilitiesResponse: Raises: TwinkleClientError: If the request fails. """ - response = http_get(self._get_url('/get_server_capabilities')) - data = self._handle_response(response) - return GetServerCapabilitiesResponse(**data) + cached = self._transport.cached_capabilities + if isinstance(cached, GetServerCapabilitiesResponse): + return cached + response = self._transport.get(self._get_url('/get_server_capabilities')) + capabilities = GetServerCapabilitiesResponse.model_validate(response.json()) + self._transport.cached_capabilities = capabilities + return capabilities # ------------------------------------------------------------------ # Training Runs @@ -224,8 +265,8 @@ def list_training_runs(self, limit: int = 20, offset: int = 0, all_users: bool = if all_users: params['all_users'] = 'true' - response = http_get(self._get_url('/training_runs'), params=params) - data = self._handle_response(response) + response = self._transport.get(self._get_url('/training_runs'), params=params) + data = response.json() return [TrainingRun(**r) for r in data.get('training_runs', [])] @@ -253,8 +294,8 @@ def list_training_runs_with_cursor( if all_users: params['all_users'] = 'true' - response = http_get(self._get_url('/training_runs'), params=params) - data = self._handle_response(response) + response = self._transport.get(self._get_url('/training_runs'), params=params) + data = response.json() runs = [TrainingRun(**r) for r in data.get('training_runs', [])] cursor = Cursor(**data.get('cursor', {})) @@ -273,8 +314,8 @@ def get_training_run(self, run_id: str) -> TrainingRun: Raises: TwinkleClientError: If run not found or access denied. """ - response = http_get(self._get_url(f'/training_runs/{run_id}')) - data = self._handle_response(response) + response = self._transport.get(self._get_url(f'/training_runs/{run_id}')) + data = response.json() return TrainingRun(**data) # ------------------------------------------------------------------ @@ -294,8 +335,8 @@ def list_checkpoints(self, run_id: str) -> list[Checkpoint]: Raises: TwinkleClientError: If run not found or access denied. """ - response = http_get(self._get_url(f'/training_runs/{run_id}/checkpoints')) - data = self._handle_response(response) + response = self._transport.get(self._get_url(f'/training_runs/{run_id}/checkpoints')) + data = response.json() return [Checkpoint(**c) for c in data.get('checkpoints', [])] def get_checkpoint_path(self, run_id: str, checkpoint_id: str) -> ParsedCheckpointTwinklePath: @@ -313,8 +354,8 @@ def get_checkpoint_path(self, run_id: str, checkpoint_id: str) -> ParsedCheckpoi Raises: TwinkleClientError: If checkpoint not found or access denied. """ - response = http_get(self._get_url(f'/checkpoint_path/{run_id}/{checkpoint_id}')) - data = self._handle_response(response) + response = self._transport.get(self._get_url(f'/checkpoint_path/{run_id}/{checkpoint_id}')) + data = response.json() return ParsedCheckpointTwinklePath( path=data.get('path', ''), twinkle_path=data.get('twinkle_path', ''), @@ -354,8 +395,8 @@ def delete_checkpoint(self, run_id: str, checkpoint_id: str) -> DeleteCheckpoint TwinkleClientError: If checkpoint not found or access denied. """ url = self._get_url(f'/training_runs/{run_id}/checkpoints/{checkpoint_id}') - response = http_delete(url) - data = self._handle_response(response) + response = self._transport.delete(url) + data = response.json() return DeleteCheckpointResponse(**data) # ------------------------------------------------------------------ @@ -376,8 +417,8 @@ def get_weights_info(self, twinkle_path: str) -> WeightsInfoResponse: Raises: TwinkleClientError: If weights not found or access denied. """ - response = http_post(self._get_url('/weights_info'), json_data={'twinkle_path': twinkle_path}) - data = self._handle_response(response) + response = self._transport.post(self._get_url('/weights_info'), json_data={'twinkle_path': twinkle_path}) + data = response.json() return WeightsInfoResponse(**data) # ------------------------------------------------------------------ diff --git a/src/twinkle_client/model/__init__.py b/src/twinkle_client/model/__init__.py index 94e3538a..b4c8012f 100644 --- a/src/twinkle_client/model/__init__.py +++ b/src/twinkle_client/model/__init__.py @@ -1 +1,3 @@ from .multi_lora_transformers import MultiLoraTransformersModel + +__all__ = ['MultiLoraTransformersModel'] diff --git a/src/twinkle_client/model/multi_lora_transformers.py b/src/twinkle_client/model/multi_lora_transformers.py index b2dbdf88..9009a384 100644 --- a/src/twinkle_client/model/multi_lora_transformers.py +++ b/src/twinkle_client/model/multi_lora_transformers.py @@ -5,7 +5,8 @@ from typing import Any, Dict, Optional from twinkle_client._request_builder import build_request -from twinkle_client.http import http_post, http_post_model +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport from twinkle_client.types import model as model_types from twinkle_client.types.component import DataRef @@ -35,24 +36,28 @@ class MultiLoraTransformersModel: signatures stay ``**kwargs`` and callers are unchanged. """ - def __init__(self, model_id: str, **kwargs): - """Initialize model client.""" - from twinkle_client.http import get_base_url - self.server_url = get_base_url() + def __init__( + self, + model_id: str, + *, + transport: ClientTransport | None = None, + **kwargs, + ): + """Initialize a model wrapper bound to one immutable request identity.""" + self._transport = capture_transport(transport) kwargs.pop('data_plane_url', None) if '://' in model_id: model_id = model_id.split('://')[1] self.model_id = model_id - self.server_url = f'{self.server_url}/model/{model_id}/twinkle' + self.server_url = f'{self._transport.context.base_url}/model/{model_id}/twinkle' self.adapter_name = None # Per-client monotonic sequence for idempotent dedup of stateful training ops: # the server dedups on (session_id, seq_id) so a retried grad/step call is # applied at most once. Reserved once per call and reused on retry. self._seq_counter = itertools.count(1) self._seq_lock = threading.Lock() - response = http_post(url=f'{self.server_url}/create', ) - response.raise_for_status() + self._transport.post(f'{self.server_url}/create') # ------------------------------------------------------------------ # # Request plumbing @@ -61,11 +66,10 @@ def __init__(self, model_id: str, **kwargs): def _submit(self, endpoint: str, model_cls, response_cls, **values): """Build, send, and resolve one twinkle-native request.""" body = build_request(model_cls, **values) - response = http_post_model(f'{self.server_url}/{endpoint}', body) + response = self._transport.post_model(f'{self.server_url}/{endpoint}', body) return self._await_task(response, response_cls) - @staticmethod - def _await_task(response, model_cls): + def _await_task(self, response, model_cls): """Resolve a Submit_Endpoint response through the Client_Future_Layer. Blocks until the task is terminal and returns the deserialized ``model_cls`` @@ -73,7 +77,7 @@ def _await_task(response, model_cls): Keeps every public method's synchronous signature unchanged. """ from twinkle_client._future import resolve_response - return resolve_response(response, model_cls) + return resolve_response(response, model_cls, transport=self._transport) def _next_seq_id(self) -> int: """Reserve the next monotonic seq_id for a stateful op (dedup key with session).""" @@ -122,7 +126,7 @@ def forward(self, inputs: Any, **kwargs) -> model_types.ForwardResponse: model_types.ForwardRequest, model_types.ForwardResponse, inputs=inputs, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) def forward_only(self, inputs: Any, **kwargs) -> model_types.ForwardResponse: @@ -132,7 +136,7 @@ def forward_only(self, inputs: Any, **kwargs) -> model_types.ForwardResponse: model_types.ForwardOnlyRequest, model_types.ForwardResponse, inputs=inputs, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) def forward_backward(self, inputs: Any, **kwargs) -> model_types.ForwardBackwardResponse: @@ -142,7 +146,7 @@ def forward_backward(self, inputs: Any, **kwargs) -> model_types.ForwardBackward model_types.ForwardBackwardTaskRequest, model_types.ForwardBackwardResponse, inputs=inputs, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), seq_id=self._next_seq_id(), **kwargs) @@ -152,7 +156,7 @@ def calculate_loss(self, **kwargs) -> model_types.CalculateLossResponse: 'calculate_loss', model_types.AdapterRequest, model_types.CalculateLossResponse, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) def get_train_configs(self, **kwargs) -> model_types.GetTrainConfigsResponse: @@ -161,7 +165,7 @@ def get_train_configs(self, **kwargs) -> model_types.GetTrainConfigsResponse: 'get_train_configs', model_types.AdapterRequest, model_types.GetTrainConfigsResponse, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) def backward(self, **kwargs) -> None: @@ -170,7 +174,7 @@ def backward(self, **kwargs) -> None: 'backward', model_types.AdapterRequest, None, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), seq_id=self._next_seq_id(), **kwargs) @@ -192,7 +196,7 @@ def forward_from_data_plane( model_types.DataPlaneForwardRequest, model_types.ForwardResponse, input_refs=_data_refs(inputs), - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), input_field=input_field, kwarg_fields=kwarg_fields or {}, **kwargs) @@ -213,7 +217,7 @@ def forward_only_from_data_plane( model_types.DataPlaneForwardOnlyRequest, model_types.ForwardResponse, input_refs=_data_refs(inputs), - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), input_field=input_field, kwarg_fields=kwarg_fields or {}, output_ref=output_ref.model_dump() if output_ref is not None else None, @@ -237,7 +241,7 @@ def forward_backward_from_data_plane( model_types.DataPlaneForwardRequest, model_types.ForwardBackwardResponse, input_refs=_data_refs(inputs), - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), input_field=input_field, kwarg_fields=kwarg_fields or {}, seq_id=self._next_seq_id(), @@ -250,11 +254,21 @@ def forward_backward_from_data_plane( def step(self, **kwargs) -> None: """Execute optimizer step.""" self._submit( - 'step', model_types.StepRequest, None, adapter_name=self.adapter_name, seq_id=self._next_seq_id(), **kwargs) + 'step', + model_types.StepRequest, + None, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), + seq_id=self._next_seq_id(), + **kwargs) def zero_grad(self, **kwargs) -> None: """Zero out gradients.""" - self._submit('zero_grad', model_types.AdapterRequest, None, adapter_name=self.adapter_name, **kwargs) + self._submit( + 'zero_grad', + model_types.AdapterRequest, + None, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), + **kwargs) def lr_step(self, **kwargs) -> None: """Execute learning rate scheduler step.""" @@ -262,7 +276,7 @@ def lr_step(self, **kwargs) -> None: 'lr_step', model_types.LrStepRequest, None, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), seq_id=self._next_seq_id(), **kwargs) @@ -275,7 +289,7 @@ def clip_grad_norm(self, 'clip_grad_norm', model_types.ClipGradNormRequest, model_types.ClipGradNormResponse, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), max_grad_norm=max_grad_norm, norm_type=norm_type, **kwargs) @@ -286,7 +300,7 @@ def clip_grad_and_step(self, max_grad_norm: float = 1.0, norm_type: int = 2, **k 'clip_grad_and_step', model_types.ClipGradAndStepRequest, None, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), max_grad_norm=max_grad_norm, norm_type=norm_type, seq_id=self._next_seq_id(), @@ -299,7 +313,12 @@ def clip_grad_and_step(self, max_grad_norm: float = 1.0, norm_type: int = 2, **k def set_loss(self, loss_cls: str, **kwargs) -> None: """Set the loss function.""" self._submit( - 'set_loss', model_types.SetLossRequest, None, loss_cls=loss_cls, adapter_name=self.adapter_name, **kwargs) + 'set_loss', + model_types.SetLossRequest, + None, + loss_cls=loss_cls, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), + **kwargs) def set_optimizer(self, optimizer_cls: str, **kwargs) -> None: """Set the optimizer.""" @@ -308,7 +327,7 @@ def set_optimizer(self, optimizer_cls: str, **kwargs) -> None: model_types.SetOptimizerRequest, None, optimizer_cls=optimizer_cls, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) def set_lr_scheduler(self, scheduler_cls: str, **kwargs) -> None: @@ -318,7 +337,7 @@ def set_lr_scheduler(self, scheduler_cls: str, **kwargs) -> None: model_types.SetLrSchedulerRequest, None, scheduler_cls=scheduler_cls, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) def set_template(self, template_cls: str, **kwargs) -> None: @@ -334,7 +353,7 @@ def set_template(self, template_cls: str, **kwargs) -> None: model_types.SetTemplateRequest, None, template_cls=template_cls, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) def set_processor(self, processor_cls: str, **kwargs) -> None: @@ -344,7 +363,7 @@ def set_processor(self, processor_cls: str, **kwargs) -> None: model_types.SetProcessorRequest, None, processor_cls=processor_cls, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) def add_metric(self, metric_cls: str, is_training: Optional[bool] = None, **kwargs) -> None: @@ -355,7 +374,7 @@ def add_metric(self, metric_cls: str, is_training: Optional[bool] = None, **kwar None, metric_cls=metric_cls, is_training=is_training, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) def apply_patch(self, patch_cls: str, **kwargs) -> None: @@ -365,7 +384,7 @@ def apply_patch(self, patch_cls: str, **kwargs) -> None: model_types.ApplyPatchRequest, None, patch_cls=patch_cls, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) def calculate_metric(self, is_training: bool = True, **kwargs) -> model_types.CalculateMetricResponse: @@ -375,7 +394,7 @@ def calculate_metric(self, is_training: bool = True, **kwargs) -> model_types.Ca model_types.CalculateMetricRequest, model_types.CalculateMetricResponse, is_training=is_training, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) # ------------------------------------------------------------------ # @@ -389,12 +408,18 @@ def save(self, name: str, **kwargs) -> model_types.SaveResponse: model_types.SaveRequest, model_types.SaveResponse, name=name, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) def load(self, name: str, **kwargs) -> None: """Load model checkpoint.""" - self._submit('load', model_types.LoadRequest, None, name=name, adapter_name=self.adapter_name, **kwargs) + self._submit( + 'load', + model_types.LoadRequest, + None, + name=name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), + **kwargs) def resume_from_checkpoint(self, name: str, *, resume_only_model: bool = False, **kwargs) -> Dict[str, Any]: """Resume weights (and optionally optimizer state) from a checkpoint.""" @@ -403,7 +428,7 @@ def resume_from_checkpoint(self, name: str, *, resume_only_model: bool = False, model_types.ResumeFromCheckpointRequest, model_types.TrainingProgressResponse, name=name, - adapter_name=self.adapter_name, + adapter_name=kwargs.pop('adapter_name', self.adapter_name), resume_only_model=resume_only_model, **kwargs) return progress.result diff --git a/src/twinkle_client/processor/__init__.py b/src/twinkle_client/processor/__init__.py index 677da196..c13f457e 100644 --- a/src/twinkle_client/processor/__init__.py +++ b/src/twinkle_client/processor/__init__.py @@ -1 +1,3 @@ from .base import InputProcessor + +__all__ = ['InputProcessor'] diff --git a/src/twinkle_client/processor/base.py b/src/twinkle_client/processor/base.py index 5d8d9852..d9b7430d 100644 --- a/src/twinkle_client/processor/base.py +++ b/src/twinkle_client/processor/base.py @@ -4,23 +4,32 @@ from twinkle import DeviceMesh from twinkle.data_format import InputFeature from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport class InputProcessor: """Client wrapper for InputProcessor that calls server HTTP endpoints.""" - def __init__(self, - device_mesh: Optional[DeviceMesh] = None, - padding_free: bool = False, - framework: Literal['transformers', 'megatron'] = 'transformers', - **kwargs): + def __init__( + self, + device_mesh: Optional[DeviceMesh] = None, + padding_free: bool = False, + framework: Literal['transformers', 'megatron'] = 'transformers', + *, + transport: ClientTransport | None = None, + **kwargs, + ): + self._transport = capture_transport(transport) self.processor_id = create_remote_component( 'processor', 'InputProcessor', device_mesh=device_mesh, padding_free=padding_free, framework=framework, - **kwargs) + transport=self._transport, + **kwargs, + ) def __call__(self, inputs: Union[InputFeature, List[InputFeature]], **kwargs): - return call_remote_component(self.processor_id, '__call__', inputs=inputs, **kwargs) + return call_remote_component(self.processor_id, '__call__', transport=self._transport, inputs=inputs, **kwargs) diff --git a/src/twinkle_client/py.typed b/src/twinkle_client/py.typed new file mode 100644 index 00000000..e13a090d --- /dev/null +++ b/src/twinkle_client/py.typed @@ -0,0 +1 @@ +# PEP 561 marker diff --git a/src/twinkle_client/sampler/__init__.py b/src/twinkle_client/sampler/__init__.py index 06b961b3..f0961b1e 100644 --- a/src/twinkle_client/sampler/__init__.py +++ b/src/twinkle_client/sampler/__init__.py @@ -1 +1,3 @@ from .vllm_sampler import vLLMSampler + +__all__ = ['vLLMSampler'] diff --git a/src/twinkle_client/sampler/vllm_sampler.py b/src/twinkle_client/sampler/vllm_sampler.py index 648c4b34..67d6595b 100644 --- a/src/twinkle_client/sampler/vllm_sampler.py +++ b/src/twinkle_client/sampler/vllm_sampler.py @@ -6,7 +6,8 @@ from twinkle.data_format import InputFeature, SamplingParams, Trajectory from twinkle_client._request_builder import build_request from twinkle_client.common.json_utils import json_safe -from twinkle_client.http import http_post, http_post_model +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport from twinkle_client.types.component import DataPlaneSampleRequest, DataRef, UnloadAdapterPathsRequest from twinkle_client.types.sampler import (SamplerAddAdapterRequest, SamplerAddAdapterResponse, SampleRequest, SampleResponseModel, SampleResponseModelList, SamplerSetTemplateRequest, @@ -35,38 +36,36 @@ class vLLMSampler: The server-side session (managed by TwinkleClient) keeps the sampler alive. """ - def __init__(self, model_id: str, **kwargs): - """Create the sampler instance on server.""" - from twinkle_client.http import get_base_url - self.server_url = get_base_url() + def __init__( + self, + model_id: str, + *, + transport: ClientTransport | None = None, + **kwargs, + ): + """Create the sampler instance on server with one captured transport.""" + self._transport = capture_transport(transport) from twinkle_client.data_plane import DataPlaneClient - self.data_plane = DataPlaneClient(kwargs.pop('data_plane_url', None)) + self.data_plane = DataPlaneClient(kwargs.pop('data_plane_url', None), transport=self._transport) self.adapter_name = None if '://' in model_id: model_id = model_id.split('://')[1] self.model_id = model_id - self.server_url = f'{self.server_url}/sampler/{model_id}/twinkle' - response = http_post(url=f'{self.server_url}/create', json_data=kwargs) - response.raise_for_status() + self.server_url = f'{self._transport.context.base_url}/sampler/{model_id}/twinkle' + self._transport.post(f'{self.server_url}/create', json_data=kwargs) - @staticmethod - def _await_task(response, model_cls): - """Resolve a Submit_Endpoint response through the Client_Future_Layer. - - Blocks until the task is terminal and returns the deserialized ``model_cls`` - result (or ``None``), raising ``TaskFailedError`` on a failed terminal state. - Keeps every public method's synchronous signature unchanged. - """ + def _await_task(self, response, model_cls): + """Resolve a Submit_Endpoint response through the Client_Future_Layer.""" from twinkle_client._future import resolve_response - return resolve_response(response, model_cls) + return resolve_response(response, model_cls, transport=self._transport) def add_adapter_to_sampler(self, adapter_name: str, config: PeftConfig, **kwargs) -> SamplerAddAdapterResponse: """Add a new adapter to the sampler.""" if isinstance(config, PeftConfig): config = config.__dict__ body = build_request(SamplerAddAdapterRequest, adapter_name=adapter_name, config=config, **kwargs) - response = http_post_model(f'{self.server_url}/add_adapter_to_sampler', body) + response = self._transport.post_model(f'{self.server_url}/add_adapter_to_sampler', body) self.adapter_name = adapter_name return SamplerAddAdapterResponse(**response.json()) @@ -90,7 +89,7 @@ def sample( Returns: SampleResponseModel with 'sequences' list, each containing tokens, logprobs, stop_reason. """ - response = http_post_model( + response = self._transport.post_model( f'{self.server_url}/sample', build_request( SampleRequest, @@ -134,7 +133,7 @@ def sample_to_data_plane( group_ids=group_ids, num_samples=num_samples, **source) - response = http_post_model(f'{self.server_url}/sample_to_data_plane', body) + response = self._transport.post_model(f'{self.server_url}/sample_to_data_plane', body) return self._await_task(response, DataRef) async def asample( @@ -180,17 +179,17 @@ async def asample_to_data_plane( def unload_adapter_paths(self, adapter_paths: list[str]) -> None: """Evict policy snapshots that are no longer referenced by this client.""" - http_post_model(f'{self.server_url}/unload_adapter_paths', - build_request(UnloadAdapterPathsRequest, adapter_paths=adapter_paths)) + self._transport.post_model(f'{self.server_url}/unload_adapter_paths', + build_request(UnloadAdapterPathsRequest, adapter_paths=adapter_paths)) def set_template(self, template_cls: str, adapter_name: str = '', **kwargs) -> SamplerSetTemplateResponse: """Set the template for encoding trajectories.""" body = build_request(SamplerSetTemplateRequest, template_cls=template_cls, adapter_name=adapter_name, **kwargs) - response = http_post_model(f'{self.server_url}/set_template', body) + response = self._transport.post_model(f'{self.server_url}/set_template', body) return SamplerSetTemplateResponse(**response.json()) def apply_patch(self, patch_cls: str, **kwargs) -> None: """Apply a patch to the model.""" from twinkle_client.types.model import ApplyPatchRequest body = build_request(ApplyPatchRequest, patch_cls=patch_cls, adapter_name=self.adapter_name or '', **kwargs) - http_post_model(f'{self.server_url}/apply_patch', body) + self._transport.post_model(f'{self.server_url}/apply_patch', body) diff --git a/src/twinkle_client/types/__init__.py b/src/twinkle_client/types/__init__.py index d272f9ff..bbc05e9d 100644 --- a/src/twinkle_client/types/__init__.py +++ b/src/twinkle_client/types/__init__.py @@ -24,10 +24,135 @@ from .sampler import (SampledSequenceModel, SamplerAddAdapterRequest, SamplerAddAdapterResponse, SamplerCreateResponse, SampleRequest, SampleResponseModel, SampleResponseModelList, SamplerSetTemplateRequest, SamplerSetTemplateResponse) -from .server import (CapacityInfoResponse, CheckpointPathResponse, DeleteCheckpointResponse, ErrorResponse, - GetServerCapabilitiesResponse, HealthResponse, SupportedModel, WeightsInfoRequest) +from .server import (CapacityInfoResponse, CheckpointPathResponse, ClientFeatures, DeleteCheckpointResponse, + ErrorResponse, GetServerCapabilitiesResponse, HealthResponse, ProtocolLimits, SupportedModel, + WeightsInfoRequest) from .session import CreateSessionRequest, CreateSessionResponse, SessionHeartbeatRequest, SessionHeartbeatResponse from .training import (Checkpoint, CheckpointsListResponse, CreateModelRequest, Cursor, LoraConfig, ParsedCheckpointTwinklePath, TrainingRun, TrainingRunsResponse, WeightsInfoResponse) # yapf: enable + +__all__ = [ + 'BACKEND_ONLY_KEY', + 'DataModel', + 'FieldRole', + 'ResponseModel', + 'StrictRequest', + 'backend_kwarg', + 'backend_only', + 'fields_with_role', + 'passthrough', + 'read_backend_only', + 'read_field_role', + 'ResolvedLoadPath', + 'DataAppendRequest', + 'DataGetRequest', + 'DataPlaneSampleRequest', + 'DataPutRequest', + 'DataRef', + 'DataReleaseRequest', + 'DataRowsResponse', + 'UnloadAdapterPathsRequest', + 'CORE_INPUT_KEYS', + 'VLM_TENSOR_FIELDS', + 'WireInputBatch', + 'WireInputFeature', + 'WireInputs', + 'WireMessage', + 'WireTrajectory', + 'declared_wire_keys', + 'export_batch', + 'TERMINAL_STATUSES', + 'CancelRequest', + 'CancelResponse', + 'RetrieveFutureRequest', + 'TaskEnvelope', + 'TaskStatus', + 'AdapterRequest', + 'AddAdapterRequest', + 'AddMetricRequest', + 'AddMetricResponse', + 'ApplyPatchRequest', + 'ApplyPatchResponse', + 'BackwardResponse', + 'CalculateLossResponse', + 'CalculateMetricRequest', + 'CalculateMetricResponse', + 'ClipGradAndStepRequest', + 'ClipGradAndStepResponse', + 'ClipGradNormRequest', + 'ClipGradNormResponse', + 'CreateRequest', + 'CreateResponse', + 'DataPlaneForwardOnlyRequest', + 'DataPlaneForwardRequest', + 'ForwardBackwardResponse', + 'ForwardBackwardTaskRequest', + 'ForwardOnlyRequest', + 'ForwardRequest', + 'ForwardResponse', + 'GetTrainConfigsResponse', + 'LoadRequest', + 'LoadResponse', + 'LrStepRequest', + 'LrStepResponse', + 'ModelResult', + 'OkResponse', + 'ResumeFromCheckpointRequest', + 'SaveRequest', + 'SaveResponse', + 'SetLossRequest', + 'SetLossResponse', + 'SetLrSchedulerRequest', + 'SetLrSchedulerResponse', + 'SetOptimizerRequest', + 'SetOptimizerResponse', + 'SetProcessorRequest', + 'SetProcessorResponse', + 'SetTemplateRequest', + 'SetTemplateResponse', + 'StepRequest', + 'StepResponse', + 'TrainingProgressResponse', + 'UploadToHubRequest', + 'ZeroGradResponse', + 'ProcessorCallRequest', + 'ProcessorCallResponse', + 'ProcessorCreateRequest', + 'ProcessorCreateResponse', + 'ProcessorHeartbeatRequest', + 'ProcessorHeartbeatResponse', + 'SampledSequenceModel', + 'SamplerAddAdapterRequest', + 'SamplerAddAdapterResponse', + 'SamplerCreateResponse', + 'SampleRequest', + 'SampleResponseModel', + 'SampleResponseModelList', + 'SamplerSetTemplateRequest', + 'SamplerSetTemplateResponse', + 'CapacityInfoResponse', + 'CheckpointPathResponse', + 'ClientFeatures', + 'DeleteCheckpointResponse', + 'ErrorResponse', + 'GetServerCapabilitiesResponse', + 'HealthResponse', + 'ProtocolLimits', + 'SupportedModel', + 'WeightsInfoRequest', + 'CreateSessionRequest', + 'CreateSessionResponse', + 'SessionHeartbeatRequest', + 'SessionHeartbeatResponse', + 'Checkpoint', + 'CheckpointsListResponse', + 'CreateModelRequest', + 'Cursor', + 'LoraConfig', + 'ParsedCheckpointTwinklePath', + 'TrainingRun', + 'TrainingRunsResponse', + 'WeightsInfoResponse', +] diff --git a/src/twinkle_client/types/server.py b/src/twinkle_client/types/server.py index 235032a2..e3d5159d 100644 --- a/src/twinkle_client/types/server.py +++ b/src/twinkle_client/types/server.py @@ -1,6 +1,6 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Shared Pydantic response models for the twinkle server health/error endpoints.""" -from pydantic import BaseModel +from pydantic import BaseModel, Field from typing import List from .base import ResponseModel, StrictRequest @@ -15,9 +15,26 @@ class SupportedModel(BaseModel): model_name: str +class ClientFeatures(ResponseModel): + task_envelope: bool = True + cancel: bool = False + data_plane: bool = False + full_training: bool = False + batch_retrieve: bool = False + + +class ProtocolLimits(ResponseModel): + long_poll_timeout_seconds: float | None = None + max_payload_bytes: int | None = None + max_batch_size: int | None = None + + class GetServerCapabilitiesResponse(ResponseModel): - """Response body for the /get_server_capabilities endpoint.""" + """Versioned Twinkle-native capabilities with old-server defaults.""" supported_models: List[SupportedModel] + protocol_version: int = 1 + features: ClientFeatures = Field(default_factory=ClientFeatures) + limits: ProtocolLimits = Field(default_factory=ProtocolLimits) class HealthResponse(ResponseModel): diff --git a/tests/server/static/test_client_architecture_imports.py b/tests/server/static/test_client_architecture_imports.py new file mode 100644 index 00000000..653666ac --- /dev/null +++ b/tests/server/static/test_client_architecture_imports.py @@ -0,0 +1,43 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Executable client/server package-boundary contracts.""" +from __future__ import annotations + +import ast +from pathlib import Path + +_ROOT = Path(__file__).parents[3] / 'src' +_ALLOWED_SERVER_IMPORTS = ( + 'twinkle_client.types', + 'twinkle_client.http.headers', + 'twinkle_client.common.json_utils', + 'twinkle_client.common.serialize', +) + + +def _imports(path: Path): + tree = ast.parse(path.read_text(), filename=str(path)) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + yield from (alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + yield node.module + + +def test_client_does_not_import_server(): + offenders = [] + for path in (_ROOT / 'twinkle_client').rglob('*.py'): + for module in _imports(path): + if module == 'twinkle.server' or module.startswith('twinkle.server.'): + offenders.append(f'{path.relative_to(_ROOT)} -> {module}') + assert offenders == [] + + +def test_server_only_imports_shared_client_contracts(): + offenders = [] + for path in (_ROOT / 'twinkle' / 'server').rglob('*.py'): + for module in _imports(path): + if module == 'twinkle_client' or module.startswith('twinkle_client.'): + if not any(module == allowed or module.startswith(f'{allowed}.') + for allowed in _ALLOWED_SERVER_IMPORTS): + offenders.append(f'{path.relative_to(_ROOT)} -> {module}') + assert offenders == [] diff --git a/tests/server/test_gateway_services.py b/tests/server/test_gateway_services.py new file mode 100644 index 00000000..d4bab9ca --- /dev/null +++ b/tests/server/test_gateway_services.py @@ -0,0 +1,93 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +from __future__ import annotations + +import ast +import asyncio +from pathlib import Path + +from twinkle.server.gateway import services + + +class _State: + + def __init__(self, records): + self.records = list(records) + + async def get_future(self, request_id): + return self.records.pop(0) + + +async def _no_sleep(_seconds): + return None + + +def test_poll_future_returns_canonical_terminal_record(monkeypatch): + monkeypatch.setattr(services, 'long_poll_window', lambda: 10) + monkeypatch.setattr(services, 'retrieve_poll_interval', lambda: 0) + monkeypatch.setattr(services.asyncio, 'sleep', _no_sleep) + state = _State([None, {'status': 'running'}, {'status': 'completed', 'result': None}]) + + outcome = asyncio.run(services.poll_future(state, 'request-1')) + + assert outcome.timed_out is False + assert outcome.record == {'status': 'completed', 'result': None} + + +def test_gateway_services_do_not_import_protocol_models(): + path = Path(services.__file__) + tree = ast.parse(path.read_text(), filename=str(path)) + imports = {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.module} + assert not any(module == 'tinker.types' or module.startswith('twinkle_client.types') for module in imports) + + +class _FakeGateway: + """Minimal ``GatewayServer`` stand-in; ``poll_future`` is patched so state is unused.""" + + state = None + + +def _parity_client(monkeypatch, canonical_record): + """Register both real wire adapters on one app, feeding both the same canonical record.""" + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from twinkle.server.gateway import tinker_handlers, twinkle_handlers + + async def _fixed_poll(_state, _request_id): + return services.FuturePollResult(record=canonical_record, timed_out=False) + + app = FastAPI() + monkeypatch.setattr(tinker_handlers, 'poll_future', _fixed_poll) + monkeypatch.setattr(twinkle_handlers, 'poll_future', _fixed_poll) + tinker_handlers._register_tinker_routes(app, lambda: _FakeGateway()) + twinkle_handlers._register_twinkle_routes(app, lambda: _FakeGateway()) + return TestClient(app) + + +def test_same_completed_record_diverges_into_protocol_specific_wire_shapes(monkeypatch): + """One canonical ``completed`` record -> Tinker raw result vs Twinkle TaskEnvelope.""" + client = _parity_client(monkeypatch, {'status': 'completed', 'result': {'loss': 1.0}}) + + tinker_resp = client.post('/retrieve_future', json={'request_id': 'r1'}) + twinkle_resp = client.post('/twinkle/retrieve_future', json={'request_id': 'r1'}) + + assert tinker_resp.status_code == 200 + assert twinkle_resp.status_code == 200 + # Tinker returns the raw result payload; Twinkle wraps it in a canonical envelope. + assert tinker_resp.json() == {'loss': 1.0} + twinkle_body = twinkle_resp.json() + assert twinkle_body['request_id'] == 'r1' + assert twinkle_body['status'] == 'completed' + assert twinkle_body['result'] == {'loss': 1.0} + + +def test_completed_null_result_keeps_the_two_protocols_divergent(monkeypatch): + """The load-bearing difference: null result is a 500 for Tinker but a valid 200 envelope for Twinkle.""" + client = _parity_client(monkeypatch, {'status': 'completed', 'result': None}) + + tinker_resp = client.post('/retrieve_future', json={'request_id': 'r1'}) + twinkle_resp = client.post('/twinkle/retrieve_future', json={'request_id': 'r1'}) + + assert tinker_resp.status_code == 500 + assert twinkle_resp.status_code == 200 + assert twinkle_resp.json()['status'] == 'completed' diff --git a/tests/twinkle_client/test_async_components.py b/tests/twinkle_client/test_async_components.py index 87264dd0..05147328 100644 --- a/tests/twinkle_client/test_async_components.py +++ b/tests/twinkle_client/test_async_components.py @@ -36,10 +36,10 @@ def _completed(result): def _recorder(calls, result_factory): - """A ``requests.post`` stand-in that records the URL and the decoded JSON body.""" + """A Session.post stand-in that records the URL and decoded JSON body.""" - def post(url, headers=None, data=None, timeout=None, **_kwargs): - body = json.loads(data) if data else {} + def post(url, headers=None, data=None, timeout=None, **kwargs): + body = json.loads(data) if data else kwargs.get('json') or {} calls.append((url, body)) return _Response(result_factory(url)) @@ -47,12 +47,12 @@ def post(url, headers=None, data=None, timeout=None, **_kwargs): def _patch_transport(monkeypatch, calls, result_factory): - import twinkle_client.http as http_module - import twinkle_client.http.client as http_client - monkeypatch.setattr(http_module, 'get_base_url', lambda: 'http://server/api/v1') - monkeypatch.setattr(http_client, 'get_base_url', lambda: 'http://server/api/v1') - monkeypatch.setattr(http_client, 'get_api_key', lambda: 'test-key') - monkeypatch.setattr(http_client.requests, 'post', _recorder(calls, result_factory)) + from twinkle_client.http import ClientContext, ClientTransport + from twinkle_client.http.context import set_default_transport + + transport = ClientTransport(ClientContext(base_url='http://server', api_key='test-key')) + monkeypatch.setattr(transport._session, 'post', _recorder(calls, result_factory)) + set_default_transport(transport) def test_model_forward_backward_sends_multiple_data_refs(monkeypatch) -> None: diff --git a/tests/twinkle_client/test_future_layer.py b/tests/twinkle_client/test_future_layer.py index d8cf1264..9b389680 100644 --- a/tests/twinkle_client/test_future_layer.py +++ b/tests/twinkle_client/test_future_layer.py @@ -42,7 +42,7 @@ def _running(): def test_terminal_submit_issues_no_retrieve(monkeypatch): """R8#1: a task terminal in the submit envelope makes zero retrieve calls.""" - def _boom(_request_id): + def _boom(_request_id, _transport): raise AssertionError('retrieve must not be called for a terminal submit') monkeypatch.setattr(_future, '_post_retrieve', _boom) @@ -52,7 +52,7 @@ def _boom(_request_id): def test_terminal_submit_failure_raises_taskfailed_with_payload(monkeypatch): """Property 0: a failure in the submit envelope raises TaskFailedError, payload intact.""" - monkeypatch.setattr(_future, '_post_retrieve', lambda _r: pytest.fail('no retrieve')) + monkeypatch.setattr(_future, '_post_retrieve', lambda _r, _transport: pytest.fail('no retrieve')) with pytest.raises(TaskFailedError) as exc: _future.resolve(_failed(), model_cls=_Model) assert exc.value.error == 'boom' @@ -64,13 +64,13 @@ def test_terminal_submit_failure_raises_taskfailed_with_payload(monkeypatch): def test_model_cls_none_returns_none_result(monkeypatch): """R4#9: a method that returned None before still returns None (not swallowed).""" - monkeypatch.setattr(_future, '_post_retrieve', lambda _r: pytest.fail('no retrieve')) + monkeypatch.setattr(_future, '_post_retrieve', lambda _r, _transport: pytest.fail('no retrieve')) assert _future.resolve(_completed(None), model_cls=None) is None def test_non_terminal_submit_polls_until_terminal(monkeypatch): replies = [_running(), _running(), _completed({'ok': 1})] - monkeypatch.setattr(_future, '_post_retrieve', lambda _r: replies.pop(0)) + monkeypatch.setattr(_future, '_post_retrieve', lambda _r, _transport: replies.pop(0)) out = _future.resolve(_running(), model_cls=_Model) assert out.result == {'ok': 1} assert replies == [] @@ -78,7 +78,7 @@ def test_non_terminal_submit_polls_until_terminal(monkeypatch): def test_404_is_bounded_then_raises_record_lost(monkeypatch): - def _always_404(_request_id): + def _always_404(_request_id, _transport): e = requests.HTTPError('404') e.status_code = 404 raise e @@ -91,7 +91,7 @@ def _always_404(_request_id): def test_transport_5xx_is_bounded_then_reraises(monkeypatch): monkeypatch.setattr(_future.time, 'sleep', lambda _s: None) # no real backoff sleeps - def _always_503(_request_id): + def _always_503(_request_id, _transport): e = requests.HTTPError('503') e.status_code = 503 raise e @@ -103,7 +103,7 @@ def _always_503(_request_id): def test_non_retryable_4xx_reraises_immediately(monkeypatch): - def _400(_request_id): + def _400(_request_id, _transport): e = requests.HTTPError('400') e.status_code = 400 raise e @@ -114,7 +114,7 @@ def _400(_request_id): def test_total_timeout_raises_wait_timeout(monkeypatch): - monkeypatch.setattr(_future, '_post_retrieve', lambda _r: _running()) + monkeypatch.setattr(_future, '_post_retrieve', lambda _r, _transport: _running()) with pytest.raises(TaskWaitTimeoutError) as exc: _future.resolve(_running(), model_cls=_Model, total_timeout=0.0) assert exc.value.request_id == 'r' @@ -124,7 +124,7 @@ def test_success_resets_both_retry_counters(monkeypatch): """R4#8: a successful reply zeroes both counters, so intermittent 404s never sum up.""" seq = [] - def _mixed(_request_id): + def _mixed(_request_id, _transport): seq.append(1) n = len(seq) if n in (1, 2, 4, 5): # 404s interleaved with a success at n==3 diff --git a/tests/twinkle_client/test_transport.py b/tests/twinkle_client/test_transport.py new file mode 100644 index 00000000..43b6c34c --- /dev/null +++ b/tests/twinkle_client/test_transport.py @@ -0,0 +1,149 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +from __future__ import annotations + +from twinkle_client.http import ClientContext, ClientTransport +from twinkle_client.http.context import set_default_transport +from twinkle_client.manager import TwinkleClient + + +class _Response: + ok = True + status_code = 200 + url = 'http://server' + text = '' + + def json(self): + return {} + + +class _Session: + + def __init__(self): + self.calls = [] + self.closed = False + + def post(self, url, **kwargs): + self.calls.append(('post', url, kwargs)) + return _Response() + + def get(self, url, **kwargs): + self.calls.append(('get', url, kwargs)) + return _Response() + + def delete(self, url, **kwargs): + self.calls.append(('delete', url, kwargs)) + return _Response() + + def close(self): + self.closed = True + + +def _transport(name: str) -> ClientTransport: + return ClientTransport( + ClientContext( + base_url=f'http://{name}', + api_key=f'{name}-key', + session_id=f'{name}-session', + routing_id=f'{name}-routing', + ), + session=_Session(), + ) + + +def test_context_normalizes_url_and_transport_builds_stable_headers(): + transport = _transport('alpha') + transport.post('/resource') + _, url, kwargs = transport._session.calls[-1] + + assert url == 'http://alpha/api/v1/resource' + assert kwargs['headers']['Authorization'] == 'Bearer alpha-key' + assert kwargs['headers']['X-Twinkle-Session-Id'] == 'alpha-session' + assert kwargs['headers']['x-request-id'] == 'alpha-routing' + + +def test_wrapper_captures_default_once_and_factory_is_explicit(): + from twinkle_client.model import MultiLoraTransformersModel + + transport_a = _transport('alpha') + transport_b = _transport('beta') + set_default_transport(transport_a) + legacy_model = MultiLoraTransformersModel('model') + set_default_transport(transport_b) + + client_a = TwinkleClient(transport=transport_a) + factory_model = client_a.model('factory-model') + + assert legacy_model._transport is transport_a + assert factory_model._transport is transport_a + assert legacy_model.server_url.startswith('http://alpha/api/v1/') + assert factory_model.server_url.startswith('http://alpha/api/v1/') + + +def test_close_is_idempotent_and_does_not_clear_another_default(): + transport_a = _transport('alpha') + transport_b = _transport('beta') + client_a = TwinkleClient(transport=transport_a) + set_default_transport(transport_b) + + client_a.close() + client_a.close() + + assert transport_a.closed + assert transport_a._session.closed + assert not transport_b.closed + + +def test_http_public_api_has_no_legacy_context_getters_or_setters(): + import twinkle_client.http as http + + assert not ({ + 'get_base_url', + 'get_api_key', + 'get_session_id', + 'get_request_id', + 'set_base_url', + 'set_api_key', + 'set_session_id', + 'set_request_id', + } & set(http.__all__)) + + +class _CapabilitiesResponse(_Response): + + def json(self): + return {'supported_models': [{'model_name': 'm'}]} + + +class _CapabilitiesSession(_Session): + + def get(self, url, **kwargs): + self.calls.append(('get', url, kwargs)) + return _CapabilitiesResponse() + + +def _capability_client(name: str) -> TwinkleClient: + transport = ClientTransport( + ClientContext(base_url=f'http://{name}', api_key=f'{name}-key', session_id=f'{name}-session'), + session=_CapabilitiesSession(), + ) + return TwinkleClient(transport=transport) + + +def test_capability_cache_is_per_transport_and_not_process_global(): + from twinkle_client.types.server import GetServerCapabilitiesResponse + + client_a = _capability_client('alpha') + client_b = _capability_client('beta') + + first = client_a.get_server_capabilities() + again = client_a.get_server_capabilities() + + # Cached on the transport instance: the second query reuses it, no second GET. + assert isinstance(first, GetServerCapabilitiesResponse) + assert again is first + assert sum(1 for call in client_a.transport._session.calls if call[0] == 'get') == 1 + + # Isolation: caching on A must never populate another transport's cache. + assert client_b.transport.cached_capabilities is None + client_b.get_server_capabilities() + assert client_b.transport.cached_capabilities is not client_a.transport.cached_capabilities diff --git a/tests/twinkle_client/test_types_contract.py b/tests/twinkle_client/test_types_contract.py index 53ab20e7..03b5dfa6 100644 --- a/tests/twinkle_client/test_types_contract.py +++ b/tests/twinkle_client/test_types_contract.py @@ -17,6 +17,7 @@ import twinkle from twinkle.server.utils.task_queue.types import QueueState from twinkle_client.types.errors import QueueStateLiteral +from twinkle_client.types.server import GetServerCapabilitiesResponse _TWINKLE_SRC = pathlib.Path(twinkle.__file__).resolve().parent _LEGACY_PUBLIC_NAME_OVERLAP = frozenset({ @@ -47,6 +48,31 @@ def test_queue_state_literal_matches_server_enum(): assert literal_values == enum_values, (f'QueueStateLiteral {literal_values} != QueueState {enum_values}') +def test_old_capabilities_response_gets_conservative_defaults(): + response = GetServerCapabilitiesResponse.model_validate({'supported_models': []}) + assert response.protocol_version == 1 + assert response.features.task_envelope is True + assert response.features.cancel is False + assert response.features.batch_retrieve is False + + +def test_capabilities_response_ignores_future_fields(): + response = GetServerCapabilitiesResponse.model_validate({ + 'supported_models': [], + 'future_top_level': True, + 'features': { + 'cancel': True, + 'future_feature': True + }, + 'limits': { + 'max_batch_size': 8, + 'future_limit': 9 + }, + }) + assert response.features.cancel is True + assert response.limits.max_batch_size == 8 + + def _origin(module: str | None) -> str | None: """Classify an import's source module as 'tinker', 'twinkle_client', or None.""" if not module: From 7ba6761dfb5343278ffed22a5c590fdefd9a1705 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Sat, 19 Sep 2026 05:47:10 +0000 Subject: [PATCH 23/32] fix(model): PEFT 0.18.1 LoRA dtype autocast + static _ensure_lora_dtype; repair PPU full test suite - multi_lora: mirror PeftModel.__init__ adapter dtype autocast on each add_adapter slot (drop unconditional float() normalization) - transformers: make _ensure_lora_dtype a @staticmethod; update call site - align tests with PEFT 0.18.1 target-parameter shapes and transport API; loosen slow-startup/backend timeouts - refresh twinkle client cookbooks/docs for client-as-factory usage --- cookbook/client/twinkle/dpo.py | 1 - cookbook/client/twinkle/multi_modal.py | 1 - cookbook/client/twinkle/self_cognition.py | 1 - .../Server and Client/Twinkle-Client.md | 25 +++-- ...le\345\256\242\346\210\267\347\253\257.md" | 24 ++-- src/twinkle/model/multi_lora.py | 15 +++ .../model/transformers/transformers.py | 3 +- src/twinkle/server/processor/app.py | 6 +- .../utils/session_resource/processor.py | 8 +- src/twinkle/server/utils/task_queue/mixin.py | 34 +----- src/twinkle/server/utils/task_queue/worker.py | 13 +++ src/twinkle_client/_request_builder.py | 9 +- src/twinkle_client/http/client.py | 18 ++- .../model/multi_lora_transformers.py | 21 +++- tests/model/test_multi_lora_dtype.py | 2 +- .../test_multi_lora_target_parameters.py | 35 +++--- tests/sampler/test_vllm_startup_lock.py | 15 +-- .../integration/test_blocking_boundary.py | 10 +- tests/server/model/test_tinker_handlers.py | 2 +- .../evaluator/test_client_sampler.py | 50 ++++++--- .../test_vllm_sampler_tq_generation.py | 103 ++++++++++-------- 21 files changed, 221 insertions(+), 175 deletions(-) diff --git a/cookbook/client/twinkle/dpo.py b/cookbook/client/twinkle/dpo.py index a9279813..8fe64de2 100644 --- a/cookbook/client/twinkle/dpo.py +++ b/cookbook/client/twinkle/dpo.py @@ -191,7 +191,6 @@ def train(): # model.upload_to_hub( # checkpoint_dir=twinkle_path, # hub_model_id=hub_model_id, - # async_upload=False # ) # logger.info(f"Uploaded checkpoint to hub: {hub_model_id}") diff --git a/cookbook/client/twinkle/multi_modal.py b/cookbook/client/twinkle/multi_modal.py index 6783244b..55e71ecf 100644 --- a/cookbook/client/twinkle/multi_modal.py +++ b/cookbook/client/twinkle/multi_modal.py @@ -154,7 +154,6 @@ def train(): # model.upload_to_hub( # checkpoint_dir=twinkle_path, # hub_model_id=hub_model_id, - # async_upload=False # ) # logger.info(f"Uploaded checkpoint to hub: {hub_model_id}") diff --git a/cookbook/client/twinkle/self_cognition.py b/cookbook/client/twinkle/self_cognition.py index a7747389..3d304575 100644 --- a/cookbook/client/twinkle/self_cognition.py +++ b/cookbook/client/twinkle/self_cognition.py @@ -154,7 +154,6 @@ def train(): # model.upload_to_hub( # checkpoint_dir=twinkle_path, # hub_model_id=hub_model_id, - # async_upload=False # ) # logger.info(f"Uploaded checkpoint to hub: {hub_model_id}") diff --git a/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md b/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md index 598f215e..27fd1337 100644 --- a/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md +++ b/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md @@ -38,7 +38,7 @@ latest_path = client.get_latest_checkpoint_path(run_id='xxx') ## Migrating from Local Code to Remote -Migration is very simple, just replace the import path from `twinkle` to `twinkle_client`: +Keep the data-processing and training-loop code, then create the remote model through `client.model(...)` after initializing the remote client: ```python # Local training code (original) @@ -50,10 +50,13 @@ from twinkle.model import MultiLoraTransformersModel # DataLoader and Dataset can be imported from either local twinkle or remote twinkle_client from twinkle.dataloader import DataLoader # or: from twinkle_client.dataloader import DataLoader from twinkle.dataset import Dataset # or: from twinkle_client.dataset import Dataset -from twinkle_client.model import MultiLoraTransformersModel +from twinkle_client import init_twinkle_client + +client = init_twinkle_client(base_url=base_url, api_key=api_key) +model = client.model(f'ms://{base_model}') ``` -Training loops, data processing, and other logic do not need any modifications. +Training loops and data processing do not need any modifications. Prefer `client.model(...)` over constructing `MultiLoraTransformersModel(...)` directly so the model wrapper explicitly reuses the current client's transport, session, and authentication context. ## Complete Training Example (Transformers Backend) @@ -69,7 +72,6 @@ from twinkle_client import init_twinkle_client # DataLoader and Dataset can be imported from either local twinkle or remote twinkle_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset -from twinkle_client.model import MultiLoraTransformersModel logger = get_logger() @@ -118,8 +120,8 @@ dataset.encode(batched=True) # Create DataLoader dataloader = DataLoader(dataset=dataset, batch_size=4) -# Step 4: Configure model -model = MultiLoraTransformersModel(model_id=f'ms://{base_model}') +# Step 4: Create the remote model bound to the current client +model = client.model(f'ms://{base_model}') # Configure LoRA: apply low-rank adapters to all linear layers lora_config = LoraConfig(target_modules='all-linear') @@ -171,12 +173,14 @@ for epoch in range(3): logger.info(f'Saved checkpoint: {twinkle_path}') # Step 8: Upload to ModelScope Hub (optional) +# The server always uploads in the background; this call waits through the future layer +# until upload completion or failure. Do not pass async_upload or poll_interval: they +# remain only for old-call compatibility, are deprecated, and have no effect. # YOUR_USER_NAME = "your_username" # hub_model_id = f'{YOUR_USER_NAME}/twinkle-self-cognition' # model.upload_to_hub( # checkpoint_dir=twinkle_path, # hub_model_id=hub_model_id, -# async_upload=False # ) ``` @@ -223,17 +227,16 @@ from twinkle.advantage import GRPOAdvantage from twinkle.data_format import SamplingParams from twinkle.template import Qwen3_5Template from twinkle_agentic.tools.tool_manager import ToolManager -from twinkle_client.model import MultiLoraTransformersModel from twinkle_client.rollout import ClientMultiTurnRollout from twinkle_client.sampler import vLLMSampler MODEL_ID = 'ms://Qwen/Qwen3.5-4B' NUM_GENERATIONS = 2 # GRPO group size (rollout samples num_samples=1 per trajectory) -init_twinkle_client(base_url='http://127.0.0.1:8000', api_key='EMPTY_TOKEN') +client = init_twinkle_client(base_url='http://127.0.0.1:8000', api_key='EMPTY_TOKEN') -# Training model (GRPO) -model = MultiLoraTransformersModel(model_id=MODEL_ID) +# Training model (GRPO): bind the current transport, session, and authentication context +model = client.model(MODEL_ID) model.add_adapter_to_model('default', LoraConfig(target_modules='all-linear', r=16, lora_alpha=32)) model.set_loss('GRPOLoss', epsilon=0.2) model.set_optimizer('Adam', lr=1e-5) diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" index 668156d7..4a62e3ab 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" @@ -38,7 +38,7 @@ latest_path = client.get_latest_checkpoint_path(run_id='xxx') ## 从本地代码迁移到远端 -迁移非常简单，只需将 import 路径从 `twinkle` 替换为 `twinkle_client`： +迁移时保留数据处理和训练循环；初始化远端客户端后，通过 `client.model(...)` 创建显式绑定该客户端的远端模型： ```python # 本地训练代码（原始） @@ -50,10 +50,13 @@ from twinkle.model import MultiLoraTransformersModel # DataLoader 和 Dataset 使用本地 twinkle 或远端 twinkle_client 均可 from twinkle.dataloader import DataLoader # 或 from twinkle_client.dataloader import DataLoader from twinkle.dataset import Dataset # 或 from twinkle_client.dataset import Dataset -from twinkle_client.model import MultiLoraTransformersModel +from twinkle_client import init_twinkle_client + +client = init_twinkle_client(base_url=base_url, api_key=api_key) +model = client.model(f'ms://{base_model}') ``` -训练循环、数据处理等逻辑完全不需要修改。 +训练循环、数据处理等逻辑完全不需要修改；推荐使用 `client.model(...)`，而不是直接构造 `MultiLoraTransformersModel(...)`，以确保模型包装器显式复用当前客户端的 transport、会话与认证上下文。 ## 完整训练示例（Transformers 后端） @@ -69,7 +72,6 @@ from twinkle_client import init_twinkle_client # DataLoader 和 Dataset 使用本地 twinkle 或远端 twinkle_client 均可 from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset -from twinkle_client.model import MultiLoraTransformersModel logger = get_logger() @@ -118,8 +120,8 @@ dataset.encode(batched=True) # 创建 DataLoader dataloader = DataLoader(dataset=dataset, batch_size=4) -# Step 4: 配置模型 -model = MultiLoraTransformersModel(model_id=f'ms://{base_model}') +# Step 4: 通过当前 client 创建并绑定远端模型 +model = client.model(f'ms://{base_model}') # 配置 LoRA：对所有线性层应用低秩适配器 lora_config = LoraConfig(target_modules='all-linear') @@ -171,12 +173,13 @@ for epoch in range(3): logger.info(f'Saved checkpoint: {twinkle_path}') # Step 8: 上传到 ModelScope Hub（可选） +# 服务端始终在后台执行上传；当前调用会通过 future layer 等待上传完成或抛出失败。 +# 不要传 async_upload 或 poll_interval：两者仅为兼容旧调用保留，已废弃且无效果。 # YOUR_USER_NAME = "your_username" # hub_model_id = f'{YOUR_USER_NAME}/twinkle-self-cognition' # model.upload_to_hub( # checkpoint_dir=twinkle_path, # hub_model_id=hub_model_id, -# async_upload=False # ) ``` @@ -223,17 +226,16 @@ from twinkle.advantage import GRPOAdvantage from twinkle.data_format import SamplingParams from twinkle.template import Qwen3_5Template from twinkle_agentic.tools.tool_manager import ToolManager -from twinkle_client.model import MultiLoraTransformersModel from twinkle_client.rollout import ClientMultiTurnRollout from twinkle_client.sampler import vLLMSampler MODEL_ID = 'ms://Qwen/Qwen3.5-4B' NUM_GENERATIONS = 2 # GRPO group size（rollout 每条采样 num_samples=1） -init_twinkle_client(base_url='http://127.0.0.1:8000', api_key='EMPTY_TOKEN') +client = init_twinkle_client(base_url='http://127.0.0.1:8000', api_key='EMPTY_TOKEN') -# 训练模型（GRPO） -model = MultiLoraTransformersModel(model_id=MODEL_ID) +# 训练模型（GRPO）：通过当前 client 显式绑定 transport、会话与认证上下文 +model = client.model(MODEL_ID) model.add_adapter_to_model('default', LoraConfig(target_modules='all-linear', r=16, lora_alpha=32)) model.set_loss('GRPOLoss', epsilon=0.2) model.set_optimizer('Adam', lr=1e-5) diff --git a/src/twinkle/model/multi_lora.py b/src/twinkle/model/multi_lora.py index 6eb80ed2..70a77dca 100644 --- a/src/twinkle/model/multi_lora.py +++ b/src/twinkle/model/multi_lora.py @@ -151,6 +151,19 @@ def deactivate_adapter(self): def patch_target_parameters(self, module, target_parameters): self.target_parameter_manager.patch(module, target_parameters) + @staticmethod + def _autocast_adapter_dtype(peft_model, adapter_name: str) -> None: + """Apply PEFT's own adapter dtype autocast to one preallocated slot. + + ``PeftModel.__init__`` runs ``_cast_adapter_dtype`` (fp16/bf16 -> fp32) for the + slot created at construction time, but ``PeftModel.add_adapter`` does not. Calling + it explicitly keeps every preallocated slot on the same dtype PEFT would produce, + instead of forcing fp32 ourselves. It is a no-op for an fp32 base. + """ + base = getattr(peft_model, 'base_model', None) + if base is not None and hasattr(base, '_cast_adapter_dtype'): + base._cast_adapter_dtype(adapter_name=adapter_name, autocast_adapter_dtype=True) + @contextmanager def adapter(self, tenant_adapter_name: str, disable_lora: bool = False): self.activate_adapter(tenant_adapter_name) @@ -512,6 +525,7 @@ def patch(self, def _patch_peft(_module): if isinstance(_module, PeftModel): _module.add_adapter(lora_tenant.adapter_name, config, low_cpu_mem_usage=low_cpu_mem_usage) + self._autocast_adapter_dtype(_module, lora_tenant.adapter_name) else: _peft_model: PeftModel = get_peft_model( _module, config, lora_tenant.adapter_name, low_cpu_mem_usage=low_cpu_mem_usage) @@ -528,6 +542,7 @@ def _patch_megatron(_module): _config = deepcopy(config) if isinstance(_module, PeftModel): _module.add_adapter(lora_tenant.adapter_name, _config, low_cpu_mem_usage=low_cpu_mem_usage) + self._autocast_adapter_dtype(_module, lora_tenant.adapter_name) else: # TODO first wrap needs parse target_modules, need to fix later if _config.target_modules: diff --git a/src/twinkle/model/transformers/transformers.py b/src/twinkle/model/transformers/transformers.py index 5b8723ce..f7332eec 100644 --- a/src/twinkle/model/transformers/transformers.py +++ b/src/twinkle/model/transformers/transformers.py @@ -1480,7 +1480,8 @@ def _load_optimizer(self, checkpoint_dir, **kwargs): state_dict = torch.load(scheduler_path, map_location='cpu', weights_only=True) optimizer_config.lr_scheduler.load_state_dict(state_dict) - def _ensure_lora_dtype(self, model): + @staticmethod + def _ensure_lora_dtype(model): """Force LoRA parameters to use the same dtype as base model for FSDP2 compatibility.""" base_dtype = None is_npu_device = Platform.device_prefix() == 'npu' diff --git a/src/twinkle/server/processor/app.py b/src/twinkle/server/processor/app.py index b26bb323..8f7ccd46 100644 --- a/src/twinkle/server/processor/app.py +++ b/src/twinkle/server/processor/app.py @@ -36,7 +36,7 @@ class ProcessorManagement(LazyCleanupMixin, ProcessorManagerMixin): Lifecycle is handled by ProcessorManagerMixin: - Processors are registered with a session ID on creation. - - A background thread expires processors whose session has timed out. + - A background task expires processors whose session has timed out. - Per-user processor limit is enforced at registration. - Sticky session routing ensures session requests hit the same replica. """ @@ -82,8 +82,8 @@ async def _ensure_sticky(self): self._ensure_countdown_started() await self._ensure_state_cleanup_started() - def _on_processor_expired(self, processor_id: str) -> None: - """Called by the countdown thread when a processor's session expires.""" + async def _on_processor_expired(self, processor_id: str) -> None: + """Called by the countdown loop when a processor's session expires.""" self.resource_dict.pop(processor_id, None) self.unregister_resource(processor_id) diff --git a/src/twinkle/server/utils/session_resource/processor.py b/src/twinkle/server/utils/session_resource/processor.py index 8eebc051..2b75270b 100644 --- a/src/twinkle/server/utils/session_resource/processor.py +++ b/src/twinkle/server/utils/session_resource/processor.py @@ -85,11 +85,13 @@ async def _on_resource_expired(self, resource_id: str) -> None: ``_on_processor_expired`` is the supported extension point: the processor-domain name is kept deliberately so subclass authors override a - method named for processors rather than the generic base-class hook. + method named for processors rather than the generic base-class hook. It is + ``async`` to match the sibling ``AdapterManagerMixin._on_adapter_expired`` + contract, so both resource kinds expose the same extension-point shape. """ - self._on_processor_expired(resource_id) + await self._on_processor_expired(resource_id) - def _on_processor_expired(self, processor_id: str) -> None: + async def _on_processor_expired(self, processor_id: str) -> None: """Hook called when a processor's session expires. Must be overridden by inheriting classes. diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/utils/task_queue/mixin.py index 04e45445..eb02f883 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/utils/task_queue/mixin.py @@ -297,7 +297,7 @@ async def _schedule_task( self._compute_worker.ensure_queue_registered(queue_key) await self._compute_worker.ensure_started() - q = self._compute_worker.task_queues[queue_key] + q = self._compute_worker.get_queue(queue_key) await q.put( QueuedTask( request_id=request_id, @@ -321,7 +321,7 @@ async def _schedule_task( self._compute_worker.new_task_event.set() if self._task_metrics: - total_depth = sum(q.qsize() for q in self._compute_worker.task_queues.values()) + total_depth = self._compute_worker.total_queued() self._task_metrics.queue_depth.set(total_depth, tags={'deployment': self._deployment_name}) return {'request_id': request_id, 'model_id': model_id} @@ -509,36 +509,6 @@ def _schedule() -> None: self._event_loop.call_soon_threadsafe(_schedule) - def get_queue_stats(self) -> dict[str, Any]: - """Return current compute queue statistics. - - Not exposed over HTTP yet: no endpoint reads these three ``*_stats`` - helpers. Kept as the intended data source for a future observability - endpoint; if that endpoint never lands, delete them instead of leaving - them as dead reflection. - """ - return { - 'queue_size': - sum(q.qsize() for q in self._compute_worker.task_queues.values()), - 'queue_count': - len(self._compute_worker.task_queues), - 'worker_running': (self._compute_worker._worker_task is not None - and not self._compute_worker._worker_task.done()), - 'rate_limit_config': { - 'rps_limit': self._task_queue_config.rps_limit, - 'tps_limit': self._task_queue_config.tps_limit, - 'enabled': self._task_queue_config.enabled, - }, - } - - def get_rate_limit_stats(self, token: str) -> dict[str, Any]: - """Return rate-limiting stats for a user token. Not exposed over HTTP yet (see get_queue_stats).""" - return self._rate_limiter.get_stats(token) - - def get_rate_limiter_memory_stats(self) -> dict[str, Any]: - """Return memory usage statistics from the rate limiter. Not exposed over HTTP yet (see get_queue_stats).""" - return self._rate_limiter.get_memory_stats() - async def shutdown_task_queue(self) -> None: """Gracefully shut down the compute queue and release resources.""" await self._rate_limiter.stop_cleanup_task() diff --git a/src/twinkle/server/utils/task_queue/worker.py b/src/twinkle/server/utils/task_queue/worker.py index 8a473dcc..606f5003 100644 --- a/src/twinkle/server/utils/task_queue/worker.py +++ b/src/twinkle/server/utils/task_queue/worker.py @@ -102,6 +102,19 @@ def ensure_queue_registered(self, queue_key: str) -> None: if queue_key not in self.queue_order: self.queue_order.append(queue_key) + def get_queue(self, queue_key: str) -> asyncio.Queue: + """Return the registered queue for ``queue_key``. + + Callers must have registered it first via :meth:`ensure_queue_registered`. + Exposed so producers (the mixin) enqueue through a method instead of + indexing the worker's internal ``task_queues`` container directly. + """ + return self.task_queues[queue_key] + + def total_queued(self) -> int: + """Total number of pending tasks across all per-key queues.""" + return sum(q.qsize() for q in self.task_queues.values()) + # ------------------------------------------------------------------ # Metrics helpers # ------------------------------------------------------------------ diff --git a/src/twinkle_client/_request_builder.py b/src/twinkle_client/_request_builder.py index 6b1e4bfd..75c1399c 100644 --- a/src/twinkle_client/_request_builder.py +++ b/src/twinkle_client/_request_builder.py @@ -40,8 +40,13 @@ def to_wire_value(value: Any) -> Any: Anything else is passed through for the model to validate, so an unsupported type is reported by pydantic with its field path instead of by a generic error here. """ - if hasattr(value, 'processor_id'): - return value.processor_id + # A remote-component handle (InputProcessor / dataset / dataloader wrapper) + # is sent as its server-side id. Guarded on ``str`` so an unrelated object + # that merely happens to expose a ``processor_id`` attribute is not silently + # coerced to something that is not an id. + component_id = getattr(value, 'processor_id', None) + if isinstance(component_id, str): + return component_id from peft import LoraConfig from twinkle.dataset import DatasetMeta diff --git a/src/twinkle_client/http/client.py b/src/twinkle_client/http/client.py index 5f7feab4..a867bff8 100644 --- a/src/twinkle_client/http/client.py +++ b/src/twinkle_client/http/client.py @@ -5,7 +5,6 @@ import requests from collections.abc import Mapping from dataclasses import asdict, is_dataclass -from pydantic import BaseModel from typing import Any from twinkle_client.exceptions import TwinkleClientValidationError, TwinkleHTTPError @@ -23,19 +22,18 @@ def _serialize_value(value: Any) -> Any: return value if isinstance(value, bytes | bytearray | memoryview): raise TwinkleClientValidationError('Binary values are not supported by the JSON transport') - if isinstance(value, BaseModel): - return value.model_dump(mode='json') - if is_dataclass(value) and not isinstance(value, type): - return _serialize_value(asdict(value)) if isinstance(value, Mapping): return {str(key): _serialize_value(item) for key, item in value.items()} if isinstance(value, (list, tuple)): return [_serialize_value(item) for item in value] - processor_id = getattr(value, 'processor_id', None) - if isinstance(processor_id, str): - return processor_id - raise TwinkleClientValidationError( - f'Unsupported wire value {type(value).__name__}; use a Pydantic model, dataclass, or JSON value') + if is_dataclass(value) and not isinstance(value, type): + return _serialize_value(asdict(value)) + # Single source of truth for leaf/domain objects (pydantic models, remote + # component handles, DatasetMeta / LoraConfig, numpy / torch): reuse the + # request builder's converter so a value serializes identically whether it + # goes out via ``post(json_data=...)`` or via ``post_model(body=...)``. + from twinkle_client._request_builder import to_wire_value + return to_wire_value(value) def _handle_response(response: requests.Response) -> requests.Response: diff --git a/src/twinkle_client/model/multi_lora_transformers.py b/src/twinkle_client/model/multi_lora_transformers.py index 9009a384..525a68e9 100644 --- a/src/twinkle_client/model/multi_lora_transformers.py +++ b/src/twinkle_client/model/multi_lora_transformers.py @@ -57,14 +57,33 @@ def __init__( # applied at most once. Reserved once per call and reused on retry. self._seq_counter = itertools.count(1) self._seq_lock = threading.Lock() - self._transport.post(f'{self.server_url}/create') + # The server-side component is created lazily on first use rather than in + # __init__, so constructing the wrapper performs no network I/O. + self._created = False + self._create_lock = threading.Lock() # ------------------------------------------------------------------ # # Request plumbing # ------------------------------------------------------------------ # + def _ensure_created(self) -> None: + """Create the server-side model component once, on first use. + + Deferred out of ``__init__`` so construction has no side effect: a failed + ``create`` surfaces from the first operation instead of leaving a + half-initialised object published. Idempotent and thread-safe. + """ + if self._created: + return + with self._create_lock: + if self._created: + return + self._transport.post(f'{self.server_url}/create') + self._created = True + def _submit(self, endpoint: str, model_cls, response_cls, **values): """Build, send, and resolve one twinkle-native request.""" + self._ensure_created() body = build_request(model_cls, **values) response = self._transport.post_model(f'{self.server_url}/{endpoint}', body) return self._await_task(response, response_cls) diff --git a/tests/model/test_multi_lora_dtype.py b/tests/model/test_multi_lora_dtype.py index 611008a4..e8d403e0 100644 --- a/tests/model/test_multi_lora_dtype.py +++ b/tests/model/test_multi_lora_dtype.py @@ -17,6 +17,6 @@ def test_multi_lora_dtype_matches_bf16_base_before_fsdp_wrap(): assert {param.dtype for name, param in model.named_parameters() if 'lora_' in name} == {torch.float32} - TransformersModel._ensure_lora_dtype(None, model) + TransformersModel._ensure_lora_dtype(model) assert {param.dtype for name, param in model.named_parameters() if 'lora_' in name} == {torch.bfloat16} diff --git a/tests/model/test_multi_lora_target_parameters.py b/tests/model/test_multi_lora_target_parameters.py index c76854ce..7e23369a 100644 --- a/tests/model/test_multi_lora_target_parameters.py +++ b/tests/model/test_multi_lora_target_parameters.py @@ -1,15 +1,15 @@ import copy -import sys -import types - import pytest +import sys import torch +import types from peft import LoraConfig, get_peft_model from peft.utils import set_peft_model_state_dict from torch import nn print(f"sys.path: {sys.path}") + class FakePackedExperts(nn.Module): def __init__(self, num_experts=2, hidden=4, intermediate=6, *, is_transposed=False): @@ -95,11 +95,13 @@ def test_peft_target_parameter_key_shapes_for_3d_experts(): state = peft_model.state_dict() lora_shapes = {key: tuple(state[key].shape) for key in state if "lora_" in key} + # PEFT 0.18.1 follows Linear's (out_features, in_features) convention for + # target parameters: A projects input->rank and B projects rank->output. assert lora_shapes == { - "base_model.model.mlp.experts.base_layer.lora_A.default.weight": (4, 12), - "base_model.model.mlp.experts.base_layer.lora_B.default.weight": (4, 4), - "base_model.model.mlp.experts.lora_A.default.weight": (4, 4), - "base_model.model.mlp.experts.lora_B.default.weight": (6, 4), + "base_model.model.mlp.experts.base_layer.lora_A.default.weight": (4, 4), + "base_model.model.mlp.experts.base_layer.lora_B.default.weight": (12, 4), + "base_model.model.mlp.experts.lora_A.default.weight": (4, 6), + "base_model.model.mlp.experts.lora_B.default.weight": (4, 4), } @@ -122,10 +124,7 @@ def test_target_parameter_multi_lora_updates_only_active_adapter(): manager.acquire("adapter_a", "lora_0", _make_target_cfg(r=2)) manager.acquire("adapter_b", "lora_1", _make_target_cfg(r=2)) - params_before = { - name: param.detach().clone() - for name, param in manager.named_slot_parameters("adapter_b") - } + params_before = {name: param.detach().clone() for name, param in manager.named_slot_parameters("adapter_b")} opt = torch.optim.SGD(manager.parameters_for_tenant("adapter_a"), lr=0.1) with manager.adapter("adapter_a"): @@ -153,8 +152,7 @@ def test_multilora_releases_target_parameter_slot_to_initial_weights(): initial_a = { name: param.detach().clone() - for name, param in multi_lora.target_parameter_manager.named_slot_parameters("adapter_a") - if ".lora_A." in name + for name, param in multi_lora.target_parameter_manager.named_slot_parameters("adapter_a") if ".lora_A." in name } with torch.no_grad(): @@ -170,7 +168,8 @@ def test_multilora_releases_target_parameter_slot_to_initial_weights(): else: assert torch.count_nonzero(param.detach()) == 0 -# Note: PEFT (Parameter-Efficient Fine-Tuning) does not natively support + +# Note: PEFT (Parameter-Efficient Fine-Tuning) does not natively support # installing multiple LoRA slots on target parameters. # def test_target_parameter_state_dict_loads_with_peft(): # from twinkle.model.multi_lora_target_parameters import TargetParameterLoraManager @@ -296,11 +295,3 @@ def test_multilora_transformers_installs_target_parameters_once(): pass else: raise AssertionError("different target_parameters should be rejected") - -# Run in the local environment. -if __name__ == "__main__": - assert test_peft_target_parameter_key_shapes_for_3d_experts() == True - assert test_target_parameter_multi_lora_updates_only_active_adapter() == True - assert test_multilora_releases_target_parameter_slot_to_initial_weights() == True - assert test_multilora_state_dict_round_trips_target_parameters() == True - assert test_multilora_transformers_installs_target_parameters_once() == True diff --git a/tests/sampler/test_vllm_startup_lock.py b/tests/sampler/test_vllm_startup_lock.py index d07ad74a..06e5dc47 100644 --- a/tests/sampler/test_vllm_startup_lock.py +++ b/tests/sampler/test_vllm_startup_lock.py @@ -1,6 +1,5 @@ import multiprocessing import os - import pytest from twinkle.utils.parallel import PosixFileLock @@ -9,7 +8,7 @@ def _hold_startup_lock(lock_path: str, acquired, release) -> None: with PosixFileLock(lock_path): acquired.set() - if not release.wait(timeout=5): + if not release.wait(timeout=30): raise TimeoutError('test did not release vLLM startup lock') @@ -33,21 +32,23 @@ def test_vllm_engine_startup_is_serialized(tmp_path): try: first.start() - assert first_acquired.wait(timeout=5) + assert first_acquired.wait(timeout=30) second.start() - assert second_started.wait(timeout=5) + assert second_started.wait(timeout=30) assert not second_acquired.wait(timeout=0.2) release_first.set() - assert second_acquired.wait(timeout=5) + assert second_acquired.wait(timeout=30) finally: release_first.set() for process in (first, second): - process.join(timeout=5) + if process.pid is None: + continue + process.join(timeout=30) if process.is_alive(): process.terminate() - process.join(timeout=5) + process.join(timeout=30) assert first.exitcode == 0 assert second.exitcode == 0 diff --git a/tests/server/integration/test_blocking_boundary.py b/tests/server/integration/test_blocking_boundary.py index 6432ad79..93732604 100644 --- a/tests/server/integration/test_blocking_boundary.py +++ b/tests/server/integration/test_blocking_boundary.py @@ -9,12 +9,11 @@ from __future__ import annotations import asyncio +import httpx +import pytest import threading import time from concurrent.futures import ThreadPoolExecutor - -import httpx -import pytest from fastapi import FastAPI from fastapi.responses import JSONResponse @@ -28,8 +27,7 @@ class _Harness(TaskQueueMixin): """Minimal holder exposing the real call_backend with a chosen gate setting.""" def __init__(self, gate_enabled: bool, *, max_workers: int | None = None) -> None: - self._backend_executor = ThreadPoolExecutor( - max_workers=max_workers, thread_name_prefix='twinkle-backend') + self._backend_executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix='twinkle-backend') self._backend_probe_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='twinkle-backend-probe') self._backend_admission = asyncio.Lock() if gate_enabled else None self._backend_poisoned = asyncio.Event() @@ -175,7 +173,7 @@ async def healthz(): return JSONResponse(status_code=503, content={'healthy': False}) try: - slow = asyncio.create_task(h.call_backend(lambda: ray.get(actor.slow.remote(), timeout=2))) + slow = asyncio.create_task(h.call_backend(lambda: ray.get(actor.slow.remote(), timeout=10))) await asyncio.sleep(0.1) start = time.monotonic() async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url='http://test') as client: diff --git a/tests/server/model/test_tinker_handlers.py b/tests/server/model/test_tinker_handlers.py index 66a9ff98..52079198 100644 --- a/tests/server/model/test_tinker_handlers.py +++ b/tests/server/model/test_tinker_handlers.py @@ -1,8 +1,8 @@ import pytest -from unittest.mock import AsyncMock, MagicMock, patch from fastapi import FastAPI from starlette.requests import Request from tinker import types +from unittest.mock import AsyncMock, MagicMock, patch from twinkle.server.model.tinker_handlers import _register_tinker_routes diff --git a/tests/twinkle_agentic/evaluator/test_client_sampler.py b/tests/twinkle_agentic/evaluator/test_client_sampler.py index a542f018..91fec847 100644 --- a/tests/twinkle_agentic/evaluator/test_client_sampler.py +++ b/tests/twinkle_agentic/evaluator/test_client_sampler.py @@ -1,26 +1,46 @@ from twinkle.data_format import SamplingParams +from twinkle_client.http import ClientContext, ClientTransport from twinkle_client.sampler.vllm_sampler import vLLMSampler -def test_http_sampler_serializes_sampling_params_dataclass_once(monkeypatch): - request = {} +class _Response: + + def __init__(self, payload): + self._payload = payload + self.ok = True + self.status_code = 200 + + def json(self): + return self._payload + - class Response: - def raise_for_status(self): - pass +class _Session: - def json(self): - return {'samples': []} + def __init__(self, request): + self.request = request - def post(*, url, json_data): - request['url'] = url - request['body'] = json_data - return Response() + def post(self, url, *, data=None, json=None, **_kwargs): + self.request['url'] = url + if data is not None: + import json as json_module + self.request['body'] = json_module.loads(data) + else: + self.request['body'] = json + if url.endswith('/create'): + return _Response({}) + return _Response({'request_id': 'test', 'status': 'completed', 'result': {'samples': []}}) - import twinkle_client.sampler.vllm_sampler as module - monkeypatch.setattr(module, 'http_post', post) - sampler = object.__new__(vLLMSampler) - sampler.server_url = 'http://example/sampler/model/twinkle' + def close(self): + pass + + +def test_http_sampler_serializes_sampling_params_dataclass_once(): + request = {} + transport = ClientTransport( + ClientContext(base_url='http://example', api_key='test'), + session=_Session(request), + ) + sampler = vLLMSampler('model', transport=transport) sampler.sample([{'messages': []}], SamplingParams(max_tokens=4, num_samples=2)) assert request['body']['sampling_params']['num_samples'] == 2 assert 'num_samples' not in request['body'] diff --git a/tests/twinkle_agentic/test_vllm_sampler_tq_generation.py b/tests/twinkle_agentic/test_vllm_sampler_tq_generation.py index e64669b5..3b81f387 100644 --- a/tests/twinkle_agentic/test_vllm_sampler_tq_generation.py +++ b/tests/twinkle_agentic/test_vllm_sampler_tq_generation.py @@ -3,26 +3,22 @@ import asyncio import inspect import json +import pytest import time from concurrent.futures import Future -import pytest - from twinkle import DeviceMesh from twinkle.data_format import SampledSequence, SampleResponse, SamplingParams from twinkle.infra import _dispatch_args from twinkle.server.sampler.twinkle_handlers import _await_generation from twinkle_agentic.async_rl import LoraContext from twinkle_agentic.async_rl.types import PartitionAdmission, PromptGroup, RolloutPolicy -from twinkle_agentic.async_rl.vllm_sampler_tq import ( - VLLMSamplerTQ, - _GeneratedSample, - _PromptGroupRolloutStats, - _dispatch_generation, -) +from twinkle_agentic.async_rl.vllm_sampler_tq import (VLLMSamplerTQ, _dispatch_generation, _GeneratedSample, + _PromptGroupRolloutStats) class LocalActorHandle: + def __init__(self, target): self.target = target @@ -30,6 +26,7 @@ def __getattr__(self, name): method = getattr(self.target, name) class RemoteMethod: + async def remote(_, *args, **kwargs): result = method(*args, **kwargs) return await result if inspect.isawaitable(result) else result @@ -38,6 +35,7 @@ async def remote(_, *args, **kwargs): class PolicyProvider: + def __init__(self, policies): self.policies = iter(policies) self.released = [] @@ -105,10 +103,11 @@ def test_generation_dispatch_allows_one_prompt_with_multiple_dp_workers() -> Non _dispatch_generation( 3, worker_index, - ('submission', [{'input_ids': [1]}], 'params'), + ('submission', [{ + 'input_ids': [1] + }], 'params'), {}, - )[0][1] - for worker_index in range(3) + )[0][1] for worker_index in range(3) ] assert shards == [[{'input_ids': [1]}], [], []] @@ -128,7 +127,9 @@ def submit(coro): result = sampler.submit_generation( 'submission-1', - [{'input_ids': [1]}], + [{ + 'input_ids': [1] + }], SamplingParams(max_tokens=4), ) @@ -155,7 +156,11 @@ async def sample_single(feat, _params, **_kwargs): sampler._sample_single = sample_single responses = asyncio.run( sampler._generate_inputs( - [{'input_ids': [10]}, {'input_ids': [20]}], + [{ + 'input_ids': [10] + }, { + 'input_ids': [20] + }], SamplingParams(max_tokens=4), adapter_name='', adapter_path=None, @@ -214,6 +219,15 @@ def test_native_prompt_group_sampling_requires_context_manager() -> None: sampler.submit_prompt_groups([], SamplingParams(max_tokens=4)) +class _DirectGenerationService: + + def __init__(self, sampler): + self.sampler = sampler + + async def call_backend(self, fn, /, *args, **kwargs): + return await asyncio.to_thread(fn, *args, **kwargs) + + def test_server_waiter_admits_later_submission_before_first_finishes() -> None: class Sampler: @@ -237,14 +251,13 @@ def cancel_generation(self, submission_id): self.futures.pop(submission_id, None) sampler = Sampler() + service = _DirectGenerationService(sampler) async def run(): sampler.submit_generation('first') sampler.submit_generation('second') - first = asyncio.create_task( - _await_generation(sampler, 'first')) - second = asyncio.create_task( - _await_generation(sampler, 'second')) + first = asyncio.create_task(_await_generation(service, 'first', timeout=5)) + second = asyncio.create_task(_await_generation(service, 'second', timeout=5)) while len(sampler.submission_order) < 2: await asyncio.sleep(0) assert not first.done() @@ -284,8 +297,8 @@ def cancel_generation(self, _submission_id): sampler = Sampler() sampler.submit_generation('submission') - result = asyncio.run( - _await_generation(sampler, 'submission')) + service = _DirectGenerationService(sampler) + result = asyncio.run(_await_generation(service, 'submission', timeout=5)) assert result == ['completed'] assert sampler.status_calls == 2 @@ -314,11 +327,11 @@ def test_sampler_reports_submission_throughput_at_partition_or_shard_scope(dp_si context = _context() admission = PartitionAdmission(context, context.partition_id(0), 0, 2, 2, 0) groups = [ - PromptGroup(context, admission, f'{admission.partition_id}/group_{index}', {}, object()) - for index in range(2) + PromptGroup(context, admission, f'{admission.partition_id}/group_{index}', {}, object()) for index in range(2) ] class RolloutMetricsHarness: + def __init__(self): self.device_mesh = DeviceMesh.from_sizes(world_size=dp_size, dp_size=dp_size) self.events = [] @@ -375,28 +388,25 @@ def test_sampler_writes_one_atomic_rollout_file_per_prompt_group(tmp_path): sequences=[SampledSequence('stop', [20 + index], decoded=f'completion-{index}')], prompt_token_ids=[10, 11], ), - (policy,), + (policy, ), attempts=1, was_aborted=False, resumed_partial_output=False, - ) - for index in range(2) - ] - rows = [ - { - 'generation_idx': index, - 'rollout_policy_version': 7, - 'initial_policy_version': 7, - 'final_policy_version': 7, - 'rollout_policy_versions': [7], - 'rollout_adapter_path': '/tmp/adapter-v7', - 'stop_reason': 'stop', - 'logprobs': [-0.1], - } - for index in range(2) + ) for index in range(2) ] + rows = [{ + 'generation_idx': index, + 'rollout_policy_version': 7, + 'initial_policy_version': 7, + 'final_policy_version': 7, + 'rollout_policy_versions': [7], + 'rollout_adapter_path': '/tmp/adapter-v7', + 'stop_reason': 'stop', + 'logprobs': [-0.1], + } for index in range(2)] class Template: + @staticmethod def decode(token_ids, **_kwargs): return ' '.join(map(str, token_ids)) @@ -410,13 +420,8 @@ def decode(token_ids, **_kwargs): sampler._write_rollout_group('submission-2', group, generated, rows, [1.0, 0.0]) output_path = ( - tmp_path - / context.tenant_id - / context.training_run_id - / context.adapter_name - / 'policy_7' - / 'train_3-group_0.jsonl' - ) + tmp_path / context.tenant_id / context.training_run_id / context.adapter_name / 'policy_7' + / 'train_3-group_0.jsonl') records = [json.loads(line) for line in output_path.read_text().splitlines()] assert len(records) == 2 assert records[0]['submission_id'] == 'submission-2' @@ -446,7 +451,10 @@ def test_aborted_generation_restarts_from_original_prompt_when_partial_is_disabl VLLMSamplerTQ._generate_sample( sampler, context, - {'input_ids': [1, 2], 'labels': [-100, -100]}, + { + 'input_ids': [1, 2], + 'labels': [-100, -100] + }, SamplingParams(max_tokens=4, logprobs=1), multi_modal_data=None, logprobs_only=False, @@ -478,7 +486,10 @@ def test_aborted_generation_continues_from_partial_tokens_when_enabled(): VLLMSamplerTQ._generate_sample( sampler, context, - {'input_ids': [1, 2], 'labels': [-100, -100]}, + { + 'input_ids': [1, 2], + 'labels': [-100, -100] + }, SamplingParams(max_tokens=4, logprobs=1), multi_modal_data=None, logprobs_only=False, From 3b7461e908f05cc7882d98581a2746ce9f75d13f Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Sun, 20 Sep 2026 14:21:10 +0000 Subject: [PATCH 24/32] feat(server): harden request/error contracts (server-contract-hardening R6-R13) - R6: bounded fail-open session liveness via last_liveness_confirmed_at - R7: full ABC hooks + cluster-global processor lease quota (429/User) - R8: ConcurrencyError->StateBackendError(503); close() releases handle only, flush_all() for teardown; backend contract docstrings - R9: drop **kwargs pseudo-polymorphism; ModelManager quota -> 429/User - R10: Twinkle-native ErrorPayload single exit; EndpointUnavailableError moved to server/exceptions.py; no-HTTPException static guard - R11: FutureFailureRecord domain failure; protocol-boundary wire mapping - R12: client ErrorPayload parse with details/traceback, lowercase category - R13: remove unrunnable sampler 'torch' option Verified: unit/contract regression (361 passed) + full 2x2x3 E2E matrix (transformers+megatron x twinkle+tinker x sft/dpo/grpo, 12/12 passed incl. save-LoRA/state + resume). --- .gitignore | 3 + src/twinkle/server/config/application_spec.py | 2 +- src/twinkle/server/exceptions.py | 14 ++ src/twinkle/server/gateway/tinker_handlers.py | 43 ++++- .../server/gateway/twinkle_handlers.py | 23 +-- src/twinkle/server/lifecycle/envelope.py | 59 +++++-- src/twinkle/server/processor/app.py | 9 +- .../server/processor/twinkle_handlers.py | 91 ++++++---- src/twinkle/server/sampler/app.py | 11 +- .../server/sampler/twinkle_handlers.py | 10 +- src/twinkle/server/state/backend/base.py | 27 ++- .../server/state/backend/memory_backend.py | 19 ++- .../server/state/backend/redis_backend.py | 2 +- src/twinkle/server/state/base.py | 20 +-- src/twinkle/server/state/future_manager.py | 57 +++++-- src/twinkle/server/state/model_manager.py | 50 +++--- src/twinkle/server/state/models.py | 49 +++++- src/twinkle/server/state/sampling_manager.py | 8 +- src/twinkle/server/state/server_state.py | 155 +++++++++++++++++- src/twinkle/server/state/session_manager.py | 9 - .../server/utils/session_resource/adapter.py | 17 +- .../server/utils/session_resource/base.py | 78 +++++---- .../utils/session_resource/processor.py | 66 ++++---- src/twinkle/server/utils/task_errors.py | 40 +---- src/twinkle/server/utils/task_queue/mixin.py | 14 +- src/twinkle/server/utils/task_queue/worker.py | 76 ++++++--- src/twinkle/server/validation/__init__.py | 3 +- .../server/validation/backend_compat.py | 15 +- src/twinkle_client/exceptions.py | 11 +- src/twinkle_client/http/client.py | 29 +++- tests/server/config/test_server_config.py | 23 +++ tests/server/conftest.py | 4 +- tests/server/contract/test_error_wire.py | 9 +- tests/server/lifecycle/test_envelope.py | 46 ++++-- .../lifecycle/test_retrieve_endpoint.py | 19 ++- .../test_tinker_retrieve_regression.py | 11 +- .../server/session_resource/test_contract.py | 107 ++++++++++++ tests/server/state/fake_backend.py | 8 +- tests/server/state/test_error_payload.py | 32 +--- tests/server/state/test_future_lifecycle.py | 37 ++++- tests/server/state/test_managers.py | 65 +++++++- tests/server/state/test_update_atomic.py | 81 ++++++++- .../static/test_no_twinkle_http_exception.py | 42 +++++ tests/server/utils/test_task_queue_mixin.py | 20 ++- tests/server/validation/test_preflight.py | 4 +- tests/twinkle_client/test_error_parsing.py | 44 ++++- 46 files changed, 1136 insertions(+), 426 deletions(-) create mode 100644 tests/server/session_resource/test_contract.py create mode 100644 tests/server/static/test_no_twinkle_http_exception.py diff --git a/.gitignore b/.gitignore index ad9fcd68..6ba7b538 100644 --- a/.gitignore +++ b/.gitignore @@ -166,3 +166,6 @@ tests/server/config/_generated_e2e.yaml # Regenerate via `python -m tests.server.contract.update_baseline`; do not commit it. # NOTE: client_api_routes.json is the compact, COMMITTED guard -- do not ignore that one. tests/server/contract/client_api_baseline.json + +# Redis dump file produced by a local redis-server (test infra), never source. +*.rdb diff --git a/src/twinkle/server/config/application_spec.py b/src/twinkle/server/config/application_spec.py index 60792d5d..3117d65c 100644 --- a/src/twinkle/server/config/application_spec.py +++ b/src/twinkle/server/config/application_spec.py @@ -85,7 +85,7 @@ class SamplerArgs(_ArgsBase): nproc_per_node: int = 1 device_group: dict[str, Any] device_mesh: dict[str, Any] - sampler_type: Literal['mock', 'vllm', 'vllm_async', 'torch'] + sampler_type: Literal['mock', 'vllm', 'vllm_async'] engine_args: dict[str, Any] | None = None queue_config: TaskQueueConfig = Field(default_factory=TaskQueueConfig) data_plane_url: str | None = None diff --git a/src/twinkle/server/exceptions.py b/src/twinkle/server/exceptions.py index afe197d4..0d8c6437 100644 --- a/src/twinkle/server/exceptions.py +++ b/src/twinkle/server/exceptions.py @@ -90,6 +90,13 @@ class ResourceExhaustedError(TwinkleServerError): category = ErrorCategory.Server +class EndpointUnavailableError(TwinkleServerError): + """The endpoint is not implemented by this deployment's backend.""" + + error_code = 501 + category = ErrorCategory.Server + + class RequestRejectedError(TwinkleServerError): """Decision_Boundary-left failure: rejectable from the request body, deployment config, and loaded schema alone, so it is returned with a real HTTP status code @@ -133,6 +140,13 @@ class RateLimitExceededError(RequestRejectedError): category = ErrorCategory.User +class ResourceQuotaExceededError(RequestRejectedError): + """The caller exhausted a configured per-token resource quota.""" + + error_code = 429 + category = ErrorCategory.User + + class ResourceNotFoundError(RequestRejectedError): """A well-formed request names a resource (adapter / session) that is absent. diff --git a/src/twinkle/server/gateway/tinker_handlers.py b/src/twinkle/server/gateway/tinker_handlers.py index fd61d5e5..184023c1 100644 --- a/src/twinkle/server/gateway/tinker_handlers.py +++ b/src/twinkle/server/gateway/tinker_handlers.py @@ -17,8 +17,9 @@ from twinkle.hub import HubOperation from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager +from twinkle.server.state.models import FutureFailureRecord from twinkle.server.utils.auth import get_token_from_request -from twinkle.server.utils.task_errors import error_payload_from_stored +from twinkle.server.utils.task_errors import trim_traceback from twinkle.utils.logger import get_logger from .services import create_session as create_session_use_case from .services import delete_checkpoint @@ -27,6 +28,43 @@ logger = get_logger() +# Keys must equal ``state.models.FAILURE_REASON_CODES`` (guarded by test_envelope). +_TINKER_FAILURE_WIRE: dict[str, tuple[int, str]] = { + 'invalid_request': (400, 'user'), + 'request_rejected': (400, 'user'), + 'resource_not_found': (404, 'user'), + 'full_mode_busy': (409, 'user'), + 'input_tokens_exceeded': (422, 'user'), + 'batch_size_invalid': (422, 'user'), + 'rate_limit_exceeded': (429, 'user'), + 'resource_quota_exceeded': (429, 'user'), + 'cancelled': (499, 'user'), + 'endpoint_unavailable': (501, 'server'), + 'state_contention': (503, 'server'), + 'backend_gate_unavailable': (503, 'server'), + 'orphaned_replica': (503, 'server'), + 'execution_timeout': (504, 'server'), + 'deadline_exceeded': (500, 'server'), + 'internal_error': (500, 'server'), +} + + +def _tinker_error_from_failure(stored: Any, *, request_id: str) -> dict[str, Any]: + """Map domain failure state to the existing Tinker-compatible error body.""" + failure = FutureFailureRecord.model_validate(stored) + error_code, category = _TINKER_FAILURE_WIRE.get(failure.reason_code, (500, 'server')) + payload: dict[str, Any] = { + 'error': failure.message[:1024], + 'category': category, + 'error_code': error_code, + 'request_id': request_id, + } + if failure.details is not None: + payload['details'] = failure.details + if failure.diagnostic and category == 'server': + payload['traceback'] = trim_traceback(failure.diagnostic) + return payload + def _register_tinker_routes(app: FastAPI, self_fn: Callable[[], GatewayServer]) -> None: """Register all /* Tinker routes on the given FastAPI app. @@ -97,8 +135,7 @@ async def retrieve_future(request: Request, status = record.get('status') if status == 'failed': - payload = error_payload_from_stored(record.get('result'), request_id=request_id) - return payload.model_dump(mode='json', exclude_none=True) + return _tinker_error_from_failure(record.get('failure'), request_id=request_id) result = record.get('result') if result is None: diff --git a/src/twinkle/server/gateway/twinkle_handlers.py b/src/twinkle/server/gateway/twinkle_handlers.py index 94c43a84..83aa84d9 100644 --- a/src/twinkle/server/gateway/twinkle_handlers.py +++ b/src/twinkle/server/gateway/twinkle_handlers.py @@ -7,7 +7,7 @@ from __future__ import annotations from collections.abc import Callable -from fastapi import Depends, FastAPI, HTTPException, Request +from fastapi import Depends, FastAPI, Request from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -15,6 +15,7 @@ import twinkle_client.types as types from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager, validate_user_path +from twinkle.server.exceptions import RequestRejectedError, ResourceNotFoundError from twinkle.server.lifecycle.envelope import envelope_from_record from twinkle.server.lifecycle.poll_config import long_poll_window from twinkle.server.utils.auth import get_token_from_request @@ -115,7 +116,7 @@ async def session_heartbeat( ) -> types.SessionHeartbeatResponse: alive = await touch_session(self.state, body.session_id) if not alive: - raise HTTPException(status_code=404, detail='Unknown session') + raise ResourceNotFoundError('Unknown session') return types.SessionHeartbeatResponse() @app.post('/twinkle/retrieve_future', response_model=types.TaskEnvelope) @@ -141,7 +142,7 @@ async def retrieve_future( request_id = body.request_id outcome = await poll_future(self.state, request_id) if outcome.record is None: - raise HTTPException(status_code=404, detail=f'request_id {request_id} not found or expired') + raise ResourceNotFoundError(f'request_id {request_id} not found or expired') return envelope_from_record(request_id, outcome.record) @app.post('/twinkle/cancel', response_model=types.CancelResponse) @@ -169,7 +170,7 @@ async def get_training_run(request: Request, run_id: str) -> types.TrainingRun: token = get_token_from_request(request) run = get_training_run_use_case(token, 'twinkle', run_id, check_permission=True) if not run: - raise HTTPException(status_code=404, detail=f'Training run {run_id} not found or access denied') + raise ResourceNotFoundError(f'Training run {run_id} not found or access denied') return run @app.get('/twinkle/training_runs/{run_id}/checkpoints', response_model=types.CheckpointsListResponse) @@ -177,7 +178,7 @@ async def get_run_checkpoints(request: Request, run_id: str) -> types.Checkpoint token = get_token_from_request(request) response = list_checkpoints(token, 'twinkle', run_id) if response is None: - raise HTTPException(status_code=404, detail=f'Training run {run_id} not found or access denied') + raise ResourceNotFoundError(f'Training run {run_id} not found or access denied') return response @app.delete( @@ -188,11 +189,11 @@ async def delete_run_checkpoint(request: Request, run_id: str, token = get_token_from_request(request) if not validate_user_path(token, checkpoint_id): - raise HTTPException(status_code=400, detail='Invalid checkpoint path: path traversal not allowed') + raise RequestRejectedError('Invalid checkpoint path: path traversal not allowed') success = delete_checkpoint(token, 'twinkle', run_id, checkpoint_id) if not success: - raise HTTPException(status_code=404, detail=f'Checkpoint {checkpoint_id} not found or access denied') + raise ResourceNotFoundError(f'Checkpoint {checkpoint_id} not found or access denied') return types.DeleteCheckpointResponse(success=True, message=f'Checkpoint {checkpoint_id} deleted successfully') @@ -201,7 +202,7 @@ async def weights_info(request: Request, body: types.WeightsInfoRequest) -> type token = get_token_from_request(request) response = get_weights_info(token, 'twinkle', body.twinkle_path) if response is None: - raise HTTPException(status_code=404, detail=f'Weights at {body.twinkle_path} not found or access denied') + raise ResourceNotFoundError(f'Weights at {body.twinkle_path} not found or access denied') return response @app.get('/twinkle/checkpoint_path/{run_id}/{checkpoint_id:path}', response_model=types.CheckpointPathResponse) @@ -209,18 +210,18 @@ async def get_checkpoint_path(request: Request, run_id: str, checkpoint_id: str) token = get_token_from_request(request) if not validate_user_path(token, checkpoint_id): - raise HTTPException(status_code=400, detail='Invalid checkpoint path: path traversal not allowed') + raise RequestRejectedError('Invalid checkpoint path: path traversal not allowed') training_run_manager = create_training_run_manager(token, client_type='twinkle') checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') run = training_run_manager.get(run_id) if not run: - raise HTTPException(status_code=404, detail=f'Training run {run_id} not found or access denied') + raise ResourceNotFoundError(f'Training run {run_id} not found or access denied') checkpoint = checkpoint_manager.get(run_id, checkpoint_id) if not checkpoint: - raise HTTPException(status_code=404, detail=f'Checkpoint {checkpoint_id} not found') + raise ResourceNotFoundError(f'Checkpoint {checkpoint_id} not found') ckpt_dir = checkpoint_manager.get_ckpt_dir(run_id, checkpoint_id) return types.CheckpointPathResponse(path=str(ckpt_dir), twinkle_path=checkpoint.twinkle_path) diff --git a/src/twinkle/server/lifecycle/envelope.py b/src/twinkle/server/lifecycle/envelope.py index cd5b2404..6a97b3f7 100644 --- a/src/twinkle/server/lifecycle/envelope.py +++ b/src/twinkle/server/lifecycle/envelope.py @@ -10,9 +10,49 @@ from typing import Any -from twinkle.server.utils.task_errors import error_payload_from_stored +from twinkle.server.state.models import FutureFailureRecord +from twinkle.server.utils.task_errors import trim_traceback +from twinkle_client.types.errors import ErrorCategory, ErrorPayload from twinkle_client.types.lifecycle import TaskEnvelope +# Keys must equal ``state.models.FAILURE_REASON_CODES`` (guarded by test_envelope). +_FAILURE_WIRE: dict[str, tuple[int, ErrorCategory]] = { + 'invalid_request': (400, ErrorCategory.User), + 'request_rejected': (400, ErrorCategory.User), + 'resource_not_found': (404, ErrorCategory.User), + 'full_mode_busy': (409, ErrorCategory.User), + 'input_tokens_exceeded': (422, ErrorCategory.User), + 'batch_size_invalid': (422, ErrorCategory.User), + 'rate_limit_exceeded': (429, ErrorCategory.User), + 'resource_quota_exceeded': (429, ErrorCategory.User), + 'cancelled': (499, ErrorCategory.User), + 'endpoint_unavailable': (501, ErrorCategory.Server), + 'state_contention': (503, ErrorCategory.Server), + 'backend_gate_unavailable': (503, ErrorCategory.Server), + 'orphaned_replica': (503, ErrorCategory.Server), + 'execution_timeout': (504, ErrorCategory.Server), + 'deadline_exceeded': (500, ErrorCategory.Server), + 'internal_error': (500, ErrorCategory.Server), +} + + +def error_payload_from_failure(stored: Any, *, request_id: str) -> ErrorPayload: + """Map one protocol-independent failure record to Twinkle's wire model.""" + failure = FutureFailureRecord.model_validate(stored) + error_code, category = _FAILURE_WIRE.get( + failure.reason_code, + (500, ErrorCategory.Server), + ) + diagnostic = failure.diagnostic if category is ErrorCategory.Server else None + return ErrorPayload( + error=failure.message[:1024], + category=category, + error_code=error_code, + request_id=request_id, + traceback=trim_traceback(diagnostic) if diagnostic else None, + details=failure.details, + ) + def envelope_from_record( request_id: str, @@ -22,17 +62,10 @@ def envelope_from_record( ) -> TaskEnvelope: """Map a stored ``FutureRecord`` dict to the wire ``TaskEnvelope``. - Two behaviours are load-bearing: - - - The stored ``FutureRecord`` keeps a failure payload in its ``result`` field - (changing that would break state backward-compatibility). The wire split of - ``result`` / ``error`` is done here, which is why there must be exactly one - mapping point. - - A failure payload is reconstructed through Part 1's - ``error_payload_from_stored`` rather than a strict ``ErrorPayload.model_validate``. - Pre-spec records carry only ``{error, category}``; strict validation would - make retrieve return 500 for a record that should be a 200 + payload during - any rolling upgrade. + Failed and cancelled records carry a protocol-independent ``failure`` field. + This is the only place that maps those domain reasons to Twinkle's + ``ErrorPayload`` status/category vocabulary. Legacy failures embedded in + ``result`` are intentionally unsupported because that format was never merged. ``completed`` with ``result is None`` is a valid success (``step`` / ``zero_grad`` / ``lr_step`` all return ``None``); it is NOT treated as a @@ -53,7 +86,7 @@ def envelope_from_record( return TaskEnvelope( request_id=request_id, status=status, - error=error_payload_from_stored(record.get('result'), request_id=request_id), + error=error_payload_from_failure(record.get('failure'), request_id=request_id), **common, ) return TaskEnvelope( diff --git a/src/twinkle/server/processor/app.py b/src/twinkle/server/processor/app.py index 8f7ccd46..6b268d2b 100644 --- a/src/twinkle/server/processor/app.py +++ b/src/twinkle/server/processor/app.py @@ -83,9 +83,16 @@ async def _ensure_sticky(self): await self._ensure_state_cleanup_started() async def _on_processor_expired(self, processor_id: str) -> None: - """Called by the countdown loop when a processor's session expires.""" + """Remove the local processor and release its shared quota lease.""" + info = self.get_resource_info(processor_id) self.resource_dict.pop(processor_id, None) self.unregister_resource(processor_id) + if info is not None: + await self.state.release_processor_quota( + info['token'], + processor_id, + lease_seconds=self._processor_quota_lease_seconds, + ) def build_processor_app(ncpu_proc_per_node: int, diff --git a/src/twinkle/server/processor/twinkle_handlers.py b/src/twinkle/server/processor/twinkle_handlers.py index 201fd6e2..222c1dfe 100644 --- a/src/twinkle/server/processor/twinkle_handlers.py +++ b/src/twinkle/server/processor/twinkle_handlers.py @@ -54,40 +54,63 @@ async def create( session_id = get_session_id_from_request(request) processor_id = str(uuid.uuid4().hex) - # Register for lifecycle tracking (enforces per-user limit) - self.register_resource(processor_id, token, session_id) - - _kwargs.pop('remote_group', None) - _kwargs.pop('device_mesh', None) - - from twinkle_client.common.serialize import deserialize_object - resolved_kwargs = {} - for key, value in _kwargs.items(): - if isinstance(value, str) and value.startswith('pid:'): - ref_id = value[4:] - resolved_kwargs[key] = self.resource_dict[ref_id] - else: - value = deserialize_object(value) - resolved_kwargs[key] = value - - # Run processor instantiation in a thread to avoid blocking the event loop, - # which would starve the session-liveness coroutines submitted by the - # countdown thread via asyncio.run_coroutine_threadsafe. - _remote_group = self.device_group.name - _device_mesh = self.device_mesh - - def _do_create(): - return getattr(processor_module, class_type)( - remote_group=_remote_group, device_mesh=_device_mesh, instance_id=processor_id, **resolved_kwargs) - - # Span the primary processor.create op with token + session correlation. - with traced_operation( - f'processor.create.{processor_type_name}.{class_type}', attrs={ - TOKEN_ID: token, - SESSION_ID: session_id, - }): - processor = await asyncio.get_running_loop().run_in_executor(None, _do_create) - self.resource_dict[processor_id] = processor + # Reject malformed registrations before touching shared quota state. + self._validate_registration(processor_id, token, session_id) + await self.state.reserve_processor_quota( + token, + processor_id, + session_id, + limit=self._per_token_processor_limit, + lease_seconds=self._processor_quota_lease_seconds, + ) + + try: + self.register_resource(processor_id, token, session_id) + + _kwargs.pop('remote_group', None) + _kwargs.pop('device_mesh', None) + + from twinkle_client.common.serialize import deserialize_object + resolved_kwargs = {} + for key, value in _kwargs.items(): + if isinstance(value, str) and value.startswith('pid:'): + ref_id = value[4:] + resolved_kwargs[key] = self.resource_dict[ref_id] + else: + value = deserialize_object(value) + resolved_kwargs[key] = value + + # Run processor instantiation in a thread to avoid blocking the event loop, + # which would starve the session-liveness coroutines submitted by the + # countdown thread via asyncio.run_coroutine_threadsafe. + _remote_group = self.device_group.name + _device_mesh = self.device_mesh + + def _do_create(): + return getattr(processor_module, class_type)( + remote_group=_remote_group, device_mesh=_device_mesh, instance_id=processor_id, **resolved_kwargs) + + # Span the primary processor.create op with token + session correlation. + with traced_operation( + f'processor.create.{processor_type_name}.{class_type}', attrs={ + TOKEN_ID: token, + SESSION_ID: session_id, + }): + processor = await asyncio.get_running_loop().run_in_executor(None, _do_create) + self.resource_dict[processor_id] = processor + except Exception: + self.resource_dict.pop(processor_id, None) + self.unregister_resource(processor_id) + try: + await self.state.release_processor_quota( + token, + processor_id, + lease_seconds=self._processor_quota_lease_seconds, + ) + except Exception as release_error: + logger.warning('Failed to release processor quota after create failure for %s: %r', + processor_id, release_error) + raise return types.ProcessorCreateResponse(processor_id='pid:' + processor_id) @app.post('/twinkle/call', response_model=types.ProcessorCallResponse) diff --git a/src/twinkle/server/sampler/app.py b/src/twinkle/server/sampler/app.py index d38e6b0f..3d7e92a8 100644 --- a/src/twinkle/server/sampler/app.py +++ b/src/twinkle/server/sampler/app.py @@ -44,12 +44,6 @@ def _make_vllm_async_sampler(kw: dict[str, Any]) -> Any: return VLLMSamplerTQ(**kw, context_manager=None) -def _make_torch_sampler(kw: dict[str, Any]) -> Any: - from twinkle.sampler import TorchSampler # type: ignore[attr-defined] - - return TorchSampler(**kw) - - # Single validate-then-dispatch selector for the sampler backend. SAMPLER_SELECTOR = BackendSelector( 'sampler_type', @@ -57,7 +51,6 @@ def _make_torch_sampler(kw: dict[str, Any]) -> Any: 'mock': _make_mock_sampler, 'vllm': _make_vllm_sampler, 'vllm_async': _make_vllm_async_sampler, - 'torch': _make_torch_sampler, }, ) @@ -76,7 +69,7 @@ class SamplerManagement(LazyCleanupMixin, TaskQueueMixin): """Unified sampler management service. Manages: - - vLLM or Torch sampler initialization and lifecycle + - mock or vLLM sampler initialization and lifecycle - Tinker inference requests (/tinker/asample) with rate limiting via TaskQueueMixin - Twinkle inference requests (/twinkle/*) calling sampler directly - Template configuration for trajectory encoding @@ -182,7 +175,7 @@ def build_sampler_app(model_id: str, device_group: Device group configuration dict device_mesh: Device mesh configuration dict for parallelism deploy_options: Ray Serve deployment options - sampler_type: Sampler selector — ``mock`` | ``vllm`` | ``vllm_async`` | ``torch``. + sampler_type: Sampler selector — ``mock`` | ``vllm`` | ``vllm_async``. Validated up front; bad values raise :class:`ConfigError` before any side effect. engine_args: Additional engine arguments for the sampler diff --git a/src/twinkle/server/sampler/twinkle_handlers.py b/src/twinkle/server/sampler/twinkle_handlers.py index a0704d3f..f44ce57f 100644 --- a/src/twinkle/server/sampler/twinkle_handlers.py +++ b/src/twinkle/server/sampler/twinkle_handlers.py @@ -11,7 +11,7 @@ import traceback import uuid from collections.abc import Callable -from fastapi import Depends, FastAPI, HTTPException, Request +from fastapi import Depends, FastAPI, Request from fastapi.responses import StreamingResponse from typing import TYPE_CHECKING @@ -22,7 +22,7 @@ import twinkle_client.types as types from twinkle.data_format import SamplingParams -from twinkle.server.exceptions import RequestRejectedError +from twinkle.server.exceptions import EndpointUnavailableError, RequestRejectedError from twinkle.server.lifecycle.submit import backend_kwargs, resolve_twinkle_adapter_name, to_backend_inputs from twinkle.server.sampler.weights import resolve_sampler_weights from twinkle.server.telemetry.correlation import MODEL_ID @@ -277,9 +277,9 @@ async def sample_to_data_plane( whose result is the stored group's DataRef.""" token = await self._on_request_start(request) if not self.data_plane.enabled: - raise HTTPException(status_code=503, detail='sample_to_data_plane requires data_plane_url') + raise EndpointUnavailableError('sample_to_data_plane requires data_plane_url') if not callable(getattr(self.sampler, 'submit_generation', None)): - raise HTTPException(status_code=503, detail='sampler_type must be vllm_async') + raise EndpointUnavailableError('sampler_type must be vllm_async') adapter_path = None full_adapter_name = _get_twinkle_sampler_adapter_name(request, body.adapter_name) or '' @@ -419,7 +419,7 @@ async def sample_stream( try: inputs_parsed = to_backend_inputs(body.inputs, single=True) except ValueError as e: - raise HTTPException(status_code=400, detail=str(e)) + raise RequestRejectedError(str(e)) params = None if body.sampling_params: diff --git a/src/twinkle/server/state/backend/base.py b/src/twinkle/server/state/backend/base.py index e50f0f11..f801335c 100644 --- a/src/twinkle/server/state/backend/base.py +++ b/src/twinkle/server/state/backend/base.py @@ -4,16 +4,29 @@ from collections.abc import Callable from typing import Any +from twinkle.server.exceptions import StateBackendError +from twinkle_client.types.errors import ErrorCategory -class ConcurrencyError(RuntimeError): - """Raised by ``StateBackend.update_atomic`` when contention exhausts retries.""" + +class ConcurrencyError(StateBackendError): + """Atomic state update failed after exhausting the backend retry budget.""" + + error_code = 503 + category = ErrorCategory.Server class StateBackend(ABC): - """Unified interface for state storage backends. + """Unified interface for the Ray-actor and Redis state backends. + + ``close()`` releases only this process's backend handle; shared state must + survive. ``key_prefix`` has intentionally different physical meanings: + Redis prepends it to stored keys while Ray uses it only to namespace the + detached actor. Logical keys returned by ``keys()`` omit the Redis prefix, + so physical keyspaces cannot be copied directly between modes. - All state management operations go through this interface, supporting - multiple backend implementations (memory, file, Redis). + Only ``*`` wildcard matching is portable. Ray's ``fnmatch`` additionally + accepts ``?`` and character classes, while Redis uses its native glob. + ``health_check()`` always returns a strict bool and absorbs transport errors. """ @abstractmethod @@ -72,7 +85,7 @@ async def update_atomic( - The read/transform/write triple is atomic against concurrent callers on the same backend; Redis-backed implementations use WATCH+MULTI+EXEC and may raise :class:`ConcurrencyError` after exhausting internal - retries (default 3). + retries (currently 16). ``transform`` must be picklable when running against a Ray-backed backend — pass module-level functions wrapped with ``functools.partial``, @@ -90,7 +103,7 @@ async def mget(self, keys: list[str]) -> list[Any | None]: @abstractmethod async def close(self) -> None: - """Close backend connection / release resources.""" + """Release this process's handle without changing shared state.""" ... @abstractmethod diff --git a/src/twinkle/server/state/backend/memory_backend.py b/src/twinkle/server/state/backend/memory_backend.py index 4b7c2d96..433f8baa 100644 --- a/src/twinkle/server/state/backend/memory_backend.py +++ b/src/twinkle/server/state/backend/memory_backend.py @@ -103,7 +103,8 @@ async def mget(self, keys: list[str]) -> list[Any | None]: results.append(value) return results - async def close(self) -> None: + async def flush_all(self) -> None: + """Destructively clear shared state for tests and explicit admin flows.""" self._store.clear() async def health_check(self) -> bool: @@ -123,8 +124,8 @@ class RayActorBackend(StateBackend): def __init__(self, key_prefix: str = '') -> None: if not ray.is_initialized(): raise RuntimeError('RayActorBackend requires an initialized Ray runtime — call ' - 'ray.init() first, switch persistence to "file"/"redis", or ' - 'rely on the deployment launcher to start Ray.') + 'ray.init() first, switch persistence to "redis", or rely on ' + 'the deployment launcher to start Ray.') name = _actor_name(key_prefix) try: self._actor = ray.get_actor(name) @@ -169,13 +170,13 @@ async def mget(self, keys: list[str]) -> list[Any | None]: return await self._actor.mget.remote(keys) async def close(self) -> None: - await self._actor.close.remote() + """Release this process's actor handle without clearing shared state.""" + self._actor = None async def health_check(self) -> bool: try: - return await self._actor.health_check.remote() - except ray.exceptions.RayActorError: - # The actor crashed (OOM, node died). Don't silently re-create - # it — that would lose all in-memory state. Let readiness probes - # see False and the deployment owner decide to restart. + return bool(await self._actor.health_check.remote()) + except Exception: + # The actor crashed (OOM, node died) or this local handle was closed. + # Do not silently recreate it because that could hide state loss. return False diff --git a/src/twinkle/server/state/backend/redis_backend.py b/src/twinkle/server/state/backend/redis_backend.py index cc11ac3f..e6769551 100644 --- a/src/twinkle/server/state/backend/redis_backend.py +++ b/src/twinkle/server/state/backend/redis_backend.py @@ -157,6 +157,6 @@ async def close(self) -> None: async def health_check(self) -> bool: """Check if Redis is healthy and available.""" try: - return await self._client.ping() + return bool(await self._client.ping()) except Exception: return False diff --git a/src/twinkle/server/state/base.py b/src/twinkle/server/state/base.py index 8cb055ae..bfc23841 100644 --- a/src/twinkle/server/state/base.py +++ b/src/twinkle/server/state/base.py @@ -2,7 +2,7 @@ from __future__ import annotations import time -from abc import ABC, abstractmethod +from abc import ABC from datetime import datetime, timezone from pydantic import BaseModel from typing import Generic, TypeVar @@ -17,8 +17,8 @@ class BaseManager(ABC, Generic[T]): """Abstract base class for resource managers using StateBackend. - Provides common async CRUD operations and timestamp parsing. - Subclasses must implement `cleanup_expired`. + Provides common async CRUD operations and timestamp parsing. Cleanup is + deliberately not polymorphic because each manager needs different inputs. """ def __init__(self, backend: StateBackend, key_prefix: str, record_type: type[T], expiration_timeout: float): @@ -76,20 +76,6 @@ async def get_all(self) -> dict[str, T]: result[resource_id] = self._record_type.model_validate(data) return result - # ----- Cleanup ----- - - @abstractmethod - async def cleanup_expired(self, cutoff_time: float, **kwargs) -> int: - """Remove all records older than cutoff_time. - - Args: - cutoff_time: Unix timestamp; records with activity before this are removed. - - Returns: - Number of records removed. - """ - ... - # ----- Helpers ----- def _parse_timestamp(self, timestamp_str: str) -> float: diff --git a/src/twinkle/server/state/future_manager.py b/src/twinkle/server/state/future_manager.py index c12cc5e0..54bca4c7 100644 --- a/src/twinkle/server/state/future_manager.py +++ b/src/twinkle/server/state/future_manager.py @@ -5,11 +5,10 @@ import time from typing import Any -from twinkle.server.utils.task_errors import task_error_payload from twinkle.utils.logger import get_logger from .backend.base import StateBackend from .base import BaseManager -from .models import FutureRecord, _now_iso +from .models import FutureFailureRecord, FutureRecord, _now_iso logger = get_logger() @@ -26,6 +25,7 @@ def _future_record_transform( model_id: str | None, reason: str | None, result: Any, + failure: dict[str, Any] | None, queue_state: str | None, queue_state_reason: str | None, replica_id: str | None, @@ -55,6 +55,7 @@ def _future_record_transform( model_id=model_id, reason=reason, result=result, + failure=failure, queue_state=queue_state, queue_state_reason=queue_state_reason, replica_id=replica_id, @@ -71,7 +72,14 @@ def _future_record_transform( # replica_id is set at creation and is deliberately NOT overwritten here. if reason is not None: updated['reason'] = reason - if result is not None: + if new_status in _TERMINAL_STATUSES: + if new_status in ('failed', 'cancelled'): + updated['result'] = None + updated['failure'] = failure + else: + updated['failure'] = None + updated['result'] = result + elif result is not None: updated['result'] = result if queue_state is not None: updated['queue_state'] = queue_state @@ -83,7 +91,7 @@ def _future_record_transform( _CANCELLABLE_STATUSES = frozenset({'pending', 'queued'}) -def _cancel_if_not_started_transform(existing: dict | None, *, payload: dict, now: str) -> dict | None: +def _cancel_if_not_started_transform(existing: dict | None, *, failure: dict, now: str) -> dict | None: """Atomic transform: cancel iff the task has not started (pending/queued). Returns ``None`` (no change) for running/terminal/missing records so a task @@ -94,7 +102,8 @@ def _cancel_if_not_started_transform(existing: dict | None, *, payload: dict, no return None updated = dict(existing) updated['status'] = 'cancelled' - updated['result'] = payload + updated['result'] = None + updated['failure'] = failure updated['updated_at'] = now return updated @@ -114,6 +123,7 @@ async def store_status( model_id: str | None, reason: str | None = None, result: Any = None, + failure: FutureFailureRecord | None = None, queue_state: str | None = None, queue_state_reason: str | None = None, replica_id: str | None = None, @@ -128,8 +138,16 @@ async def store_status( If the result object has a ``model_dump`` method (i.e. it is a Pydantic model) it is serialized to a plain dict before storage. """ + is_failure = status in ('failed', 'cancelled') + if is_failure and failure is None: + raise ValueError(f'{status} future requires a FutureFailureRecord') + if is_failure and result is not None: + raise ValueError(f'{status} future cannot carry result') + if not is_failure and failure is not None: + raise ValueError(f'{status} future cannot carry failure') if result is not None and hasattr(result, 'model_dump'): result = result.model_dump() + failure_data = failure.model_dump() if failure is not None else None now = _now_iso() await self._backend.update_atomic( @@ -141,6 +159,7 @@ async def store_status( model_id=model_id, reason=reason, result=result, + failure=failure_data, queue_state=queue_state, queue_state_reason=queue_state_reason, replica_id=replica_id, @@ -152,15 +171,19 @@ async def store_status( async def cancel_if_pending(self, request_id: str) -> str | None: """Cancel a task iff it has not started; return the resulting status. - Writes a terminal ``cancelled`` record (carrying a user ErrorPayload) only - when the current status is pending/queued -- a running task is left alone. + Writes a terminal ``cancelled`` record with a domain failure only when + the current status is pending/queued -- a running task is left alone. Returns the record's status after the attempt, or ``None`` if there is no record for ``request_id``. """ - payload = {'error': 'Task cancelled by client', 'category': 'user', 'error_code': 499} + failure = FutureFailureRecord( + reason_code='cancelled', + message='Task cancelled by client', + attribution='user', + ) result = await self._backend.update_atomic( self._make_key(request_id), - functools.partial(_cancel_if_not_started_transform, payload=payload, now=_now_iso()), + functools.partial(_cancel_if_not_started_transform, failure=failure.model_dump(), now=_now_iso()), ) return result.get('status') if result else None @@ -211,10 +234,10 @@ async def cleanup_expired( request_id, 'failed', record.model_id, - result=task_error_payload( - 'The replica that owned this task is no longer available.', - request_id=request_id, - error_code=503, + failure=FutureFailureRecord( + reason_code='orphaned_replica', + message='The replica that owned this task is no longer available.', + attribution='server', ), replica_id=replica_id, ) @@ -227,10 +250,10 @@ async def cleanup_expired( request_id, 'failed', record.model_id, - result=task_error_payload( - 'Task exceeded the absolute survival bound without reaching a terminal state.', - request_id=request_id, - error_code=500, + failure=FutureFailureRecord( + reason_code='deadline_exceeded', + message='Task exceeded the absolute survival bound without reaching a terminal state.', + attribution='server', ), replica_id=replica_id, ) diff --git a/src/twinkle/server/state/model_manager.py b/src/twinkle/server/state/model_manager.py index e791c454..ad286c10 100644 --- a/src/twinkle/server/state/model_manager.py +++ b/src/twinkle/server/state/model_manager.py @@ -13,6 +13,7 @@ import functools import time +from twinkle.server.exceptions import ResourceQuotaExceededError from .backend.base import StateBackend from .base import BaseManager from .models import ModelRecord @@ -37,6 +38,19 @@ def _counter_delta_transform(existing: object, *, delta: int) -> int: return new if new > 0 else 0 +async def _remove_with_record(manager: 'ModelManager', model_id: str, record: ModelRecord) -> bool: + """Remove a known record without exposing it in the public method signature.""" + removed = await BaseManager.remove(manager, model_id) + if not removed: + return False + if record.token: + await manager._backend.update_atomic( + manager._token_count_key(record.token), + functools.partial(_counter_delta_transform, delta=-1), + ) + return True + + class ModelManager(BaseManager[ModelRecord]): """Manages registered models with backend-derived per-token / per-replica indexes. @@ -156,12 +170,13 @@ async def add(self, model_id: str, record: ModelRecord) -> None: record is written, so two concurrent adds with the same token cannot both observe ``limit - 1`` and both succeed (the prior count-then-add race). If the increment would exceed the limit, it is rolled back and a - ``RuntimeError`` is raised; if the record write fails, the increment is - rolled back too so the counter never drifts above the real model count. + ``ResourceQuotaExceededError`` is raised; if the record write fails, the + increment is rolled back too so the counter never drifts above the real + model count. Raises: - RuntimeError: when adding ``record`` would exceed - ``per_token_model_limit`` for ``record.token``. + ResourceQuotaExceededError: when adding ``record`` would exceed the + configured per-token model quota. """ token = record.token if not token: @@ -180,7 +195,8 @@ async def add(self, model_id: str, record: ModelRecord) -> None: # Roll the speculative increment back and reject. ``new_count - 1`` # is the count that was already present before this add. await self._backend.update_atomic(key, functools.partial(_counter_delta_transform, delta=-1)) - raise RuntimeError(f'Model limit exceeded: {new_count - 1}/{self._per_token_model_limit} models') + raise ResourceQuotaExceededError( + f'Model quota exceeded for this token: {new_count - 1}/{self._per_token_model_limit} models') try: await super().add(model_id, record) @@ -189,28 +205,18 @@ async def add(self, model_id: str, record: ModelRecord) -> None: await self._backend.update_atomic(key, functools.partial(_counter_delta_transform, delta=-1)) raise - async def remove(self, model_id: str, *, _record: ModelRecord | None = None) -> bool: - """Remove a record by ID, decrementing its owning token's counter. - - When the caller already holds the record (e.g. from a prior ``get_all``), - pass it via ``_record`` to skip the redundant backend fetch. - """ - record = _record or await self.get(model_id) + async def remove(self, model_id: str) -> bool: + """Remove a record by ID and decrement its token quota counter.""" + record = await self.get(model_id) if record is None: return False - await super().remove(model_id) - if record.token: - await self._backend.update_atomic( - self._token_count_key(record.token), - functools.partial(_counter_delta_transform, delta=-1), - ) - return True + return await _remove_with_record(self, model_id, record) # ----- Cleanup -------------------------------------------------------- # - async def cleanup_expired(self, cutoff_time: float, expired_session_ids: list[str] | None = None, **kwargs) -> int: + async def cleanup_expired(self, cutoff_time: float, expired_session_ids: list[str]) -> int: """Remove models older than ``cutoff_time`` or whose owning session expired.""" - session_set = set(expired_session_ids or []) + session_set = set(expired_session_ids) all_records = await self.get_all() expired_ids: list[str] = [] for model_id, record in all_records.items(): @@ -221,7 +227,7 @@ async def cleanup_expired(self, cutoff_time: float, expired_session_ids: list[st if created_at < cutoff_time: expired_ids.append(model_id) for model_id in expired_ids: - await self.remove(model_id, _record=all_records[model_id]) + await _remove_with_record(self, model_id, all_records[model_id]) return len(expired_ids) # ----- Backend-derived helpers --------------------------------------- # diff --git a/src/twinkle/server/state/models.py b/src/twinkle/server/state/models.py index 7b11813a..0364aa6c 100644 --- a/src/twinkle/server/state/models.py +++ b/src/twinkle/server/state/models.py @@ -3,8 +3,8 @@ import time from datetime import datetime, timezone -from pydantic import BaseModel, Field -from typing import Any +from pydantic import BaseModel, Field, model_validator +from typing import Any, Literal def _now_iso() -> str: @@ -47,6 +47,39 @@ class SamplingSessionRecord(BaseModel): created_at: str = Field(default_factory=_now_iso) +class FutureFailureRecord(BaseModel): + """Protocol-independent reason an asynchronous task failed.""" + + reason_code: str + message: str + attribution: Literal['user', 'server'] + details: list[dict[str, Any]] | None = None + diagnostic: str | None = None + + +# Canonical set of protocol-independent failure reason codes. The Twinkle and Tinker +# gateways each map this same key set to their own wire vocabularies; keeping the set +# in one place lets a consistency test catch a map that drifts out of coverage. +FAILURE_REASON_CODES: frozenset[str] = frozenset({ + 'invalid_request', + 'request_rejected', + 'resource_not_found', + 'full_mode_busy', + 'input_tokens_exceeded', + 'batch_size_invalid', + 'rate_limit_exceeded', + 'resource_quota_exceeded', + 'cancelled', + 'endpoint_unavailable', + 'state_contention', + 'backend_gate_unavailable', + 'orphaned_replica', + 'execution_timeout', + 'deadline_exceeded', + 'internal_error', +}) + + class FutureRecord(BaseModel): """Represents an async task future / request status.""" @@ -54,6 +87,7 @@ class FutureRecord(BaseModel): model_id: str | None = None reason: str | None = None result: Any = None + failure: FutureFailureRecord | None = None queue_state: str | None = None queue_state_reason: str | None = None # Replica ownership and deadline are fixed when the record is created. @@ -61,3 +95,14 @@ class FutureRecord(BaseModel): absolute_deadline: float | None = None created_at: str = Field(default_factory=_now_iso) updated_at: str = Field(default_factory=_now_iso) + + @model_validator(mode='after') + def result_and_failure_match_status(self) -> FutureRecord: + is_failure = self.status in ('failed', 'cancelled') + if is_failure and self.failure is None: + raise ValueError(f'{self.status} future requires failure') + if is_failure and self.result is not None: + raise ValueError(f'{self.status} future cannot carry result') + if not is_failure and self.failure is not None: + raise ValueError(f'{self.status} future cannot carry failure') + return self diff --git a/src/twinkle/server/state/sampling_manager.py b/src/twinkle/server/state/sampling_manager.py index 7dd535a5..a76f2456 100644 --- a/src/twinkle/server/state/sampling_manager.py +++ b/src/twinkle/server/state/sampling_manager.py @@ -18,20 +18,20 @@ def __init__(self, backend: StateBackend, expiration_timeout: float) -> None: # ----- Cleanup ----- - async def cleanup_expired(self, cutoff_time: float, expired_session_ids: list[str] | None = None, **kwargs) -> int: + async def cleanup_expired(self, cutoff_time: float, expired_session_ids: list[str]) -> int: """Remove sampling sessions that are older than cutoff_time, or whose owning session has already been expired. Args: cutoff_time: Unix timestamp threshold. - expired_session_ids: Optional list of session IDs that have just - been expired; any sampling session belonging to one of these + expired_session_ids: Session IDs that have just been expired; any + sampling session belonging to one of these sessions will also be removed regardless of its own age. Returns: Number of sampling sessions removed. """ - session_set = set(expired_session_ids or []) + session_set = set(expired_session_ids) all_records = await self.get_all() expired_ids = [] diff --git a/src/twinkle/server/state/server_state.py b/src/twinkle/server/state/server_state.py index b675f976..db443dd3 100644 --- a/src/twinkle/server/state/server_state.py +++ b/src/twinkle/server/state/server_state.py @@ -3,6 +3,7 @@ import asyncio import functools +import math import re import time import uuid @@ -10,6 +11,7 @@ from typing import Any from twinkle.server.config.persistence import PersistenceConfig +from twinkle.server.exceptions import ResourceQuotaExceededError from twinkle.server.telemetry import MetricsRegistry from twinkle.server.telemetry.correlation import (BASE_MODEL, MODEL_ID, REPLICA_ID, SAMPLING_SESSION_ID, SESSION_ID, TOKEN_ID) @@ -20,7 +22,7 @@ from .config_manager import ConfigManager from .future_manager import FutureManager from .model_manager import ModelManager -from .models import ModelRecord, SamplingSessionRecord, SessionRecord +from .models import FutureFailureRecord, ModelRecord, SamplingSessionRecord, SessionRecord from .sampling_manager import SamplingSessionManager from .session_manager import SessionManager @@ -38,6 +40,68 @@ LEADER_KEY = 'cleanup_leader' # actual backend key: 'cleanup_leader' LEASE_TTL = 30 # seconds — leader loses the lease after this without a renew LEASE_RENEW = 10 # seconds — must be < LEASE_TTL/2 so two missed renews still beat the TTL +_PROCESSOR_QUOTA_PREFIX = 'processor_quota::' + + +def _clean_processor_reservations(existing: Any, *, now: float) -> dict[str, dict[str, Any]]: + """Return only well-formed processor reservations whose leases are active.""" + if not isinstance(existing, dict): + return {} + active: dict[str, dict[str, Any]] = {} + for processor_id, reservation in existing.items(): + if not isinstance(processor_id, str) or not isinstance(reservation, dict): + continue + expires_at = reservation.get('lease_expires_at') + if isinstance(expires_at, (int, float)) and float(expires_at) > now: + active[processor_id] = dict(reservation) + return active + + +def _reserve_processor_transform( + existing: Any, + *, + processor_id: str, + session_id: str, + now: float, + lease_seconds: float, + limit: int, +) -> dict[str, dict[str, Any]]: + reservations = _clean_processor_reservations(existing, now=now) + if processor_id in reservations or len(reservations) < limit: + reservations[processor_id] = { + 'session_id': session_id, + 'lease_expires_at': now + lease_seconds, + } + return reservations + + +def _renew_processor_transform( + existing: Any, + *, + processor_id: str, + now: float, + lease_seconds: float, +) -> dict[str, dict[str, Any]]: + reservations = _clean_processor_reservations(existing, now=now) + reservation = reservations.get(processor_id) + if reservation is not None: + reservation['lease_expires_at'] = now + lease_seconds + return reservations + + +def _release_processor_transform( + existing: Any, + *, + processor_id: str, + now: float, +) -> dict[str, dict[str, Any]]: + reservations = _clean_processor_reservations(existing, now=now) + reservations.pop(processor_id, None) + return reservations + + +def _sweep_processor_transform(existing: Any, *, now: float) -> dict[str, dict[str, Any]]: + return _clean_processor_reservations(existing, now=now) def _renew_if_owner(current: str | None, *, owner: str) -> str | None: @@ -237,6 +301,85 @@ async def get_available_replica_ids(self, candidate_ids: list[str]) -> list[str] """ return await self._model_mgr.get_available_replica_ids(candidate_ids) + # ----- Processor Quota Management ----- + + @staticmethod + def _processor_quota_key(token: str) -> str: + return f'{_PROCESSOR_QUOTA_PREFIX}{token}' + + @staticmethod + def _processor_quota_ttl(lease_seconds: float) -> int: + # The key-level TTL is only stale-key hygiene. Individual entries carry + # their own deadlines and are cleaned atomically on every operation. + return max(1, math.ceil(lease_seconds * 2)) + + async def reserve_processor_quota( + self, + token: str, + processor_id: str, + session_id: str, + *, + limit: int, + lease_seconds: float, + ) -> None: + """Atomically reserve one cluster-wide processor slot for ``token``.""" + now = time.time() + reservations = await self._backend.update_atomic( + self._processor_quota_key(token), + functools.partial( + _reserve_processor_transform, + processor_id=processor_id, + session_id=session_id, + now=now, + lease_seconds=lease_seconds, + limit=limit, + ), + ttl=self._processor_quota_ttl(lease_seconds), + ) + if not isinstance(reservations, dict) or processor_id not in reservations: + raise ResourceQuotaExceededError( + f'Per-user processor quota ({limit}) reached for token {token[:8]}...') + + async def renew_processor_quota( + self, + token: str, + processor_id: str, + *, + lease_seconds: float, + ) -> bool: + """Renew an existing reservation; never recreate an expired lease.""" + now = time.time() + reservations = await self._backend.update_atomic( + self._processor_quota_key(token), + functools.partial( + _renew_processor_transform, + processor_id=processor_id, + now=now, + lease_seconds=lease_seconds, + ), + ttl=self._processor_quota_ttl(lease_seconds), + ) + return isinstance(reservations, dict) and processor_id in reservations + + async def release_processor_quota(self, token: str, processor_id: str, *, lease_seconds: float = 30.0) -> None: + """Idempotently release a processor reservation.""" + await self._backend.update_atomic( + self._processor_quota_key(token), + functools.partial(_release_processor_transform, processor_id=processor_id, now=time.time()), + ttl=self._processor_quota_ttl(lease_seconds), + ) + + async def sweep_processor_quotas(self, *, lease_seconds: float = 30.0) -> None: + """Remove expired leases from every persisted processor quota map.""" + now = time.time() + keys = await self._backend.keys(f'{_PROCESSOR_QUOTA_PREFIX}*') + for key in keys: + await self._backend.update_atomic( + key, + functools.partial(_sweep_processor_transform, now=now), + ttl=self._processor_quota_ttl(lease_seconds), + ) + # ----- Sampling Session Management ----- async def create_sampling_session(self, payload: dict[str, Any], sampling_session_id: str | None = None) -> str: @@ -316,26 +459,28 @@ async def store_future_status( model_id: str | None, reason: str | None = None, result: Any = None, + failure: FutureFailureRecord | None = None, queue_state: str | None = None, queue_state_reason: str | None = None, replica_id: str | None = None, absolute_deadline: float | None = None, ) -> None: - """Store task status with optional result. + """Store task status with either a success result or domain failure. Supports the full task lifecycle: - PENDING: Task created, waiting to be processed - QUEUED: Task in queue waiting for execution - RUNNING: Task currently executing - COMPLETED: Task completed successfully (result required) - - FAILED: Task failed with error (result contains error payload) + - FAILED: Task failed (failure contains protocol-independent details) Args: request_id: Unique identifier for the request. status: Task status string (pending/queued/running/completed/failed). model_id: Optional associated model_id. reason: Optional reason string. - result: Optional result data (used for completed/failed status). + result: Optional success result data. + failure: Optional protocol-independent failure record. queue_state: Optional queue state for tinker client (active/paused_rate_limit/paused_capacity). queue_state_reason: Optional reason for the queue state. """ @@ -345,6 +490,7 @@ async def store_future_status( model_id=model_id, reason=reason, result=result, + failure=failure, queue_state=queue_state, queue_state_reason=queue_state_reason, replica_id=replica_id, @@ -405,6 +551,7 @@ async def cleanup_expired_resources(self) -> dict[str, int]: alive_replica_ids = await self._model_mgr.get_alive_replica_ids(self.expiration_timeout) futures_removed = await self._future_mgr.cleanup_expired(cutoff_time, alive_replica_ids=alive_replica_ids) + await self.sweep_processor_quotas() return { 'sessions': sessions_removed, diff --git a/src/twinkle/server/state/session_manager.py b/src/twinkle/server/state/session_manager.py index 1bc901b0..0e4beeb3 100644 --- a/src/twinkle/server/state/session_manager.py +++ b/src/twinkle/server/state/session_manager.py @@ -97,12 +97,3 @@ async def remove_many(self, ids: list[str]) -> int: if await self.remove(session_id): removed += 1 return removed - - async def cleanup_expired(self, cutoff_time: float, **kwargs) -> int: - """Remove sessions whose last activity is older than ``cutoff_time``. - - Returns: - Number of sessions removed. - """ - _, removed = await self.collect_and_remove_expired(cutoff_time) - return removed diff --git a/src/twinkle/server/utils/session_resource/adapter.py b/src/twinkle/server/utils/session_resource/adapter.py index 5f51a722..392a05bf 100644 --- a/src/twinkle/server/utils/session_resource/adapter.py +++ b/src/twinkle/server/utils/session_resource/adapter.py @@ -11,6 +11,8 @@ """ from __future__ import annotations +from abc import abstractmethod + from twinkle.utils.logger import get_logger from .base import SessionResourceMixin @@ -64,19 +66,10 @@ async def _on_resource_expired(self, resource_id: str) -> None: """ await self._on_adapter_expired(resource_id) + @abstractmethod async def _on_adapter_expired(self, adapter_name: str) -> None: - """Hook method called when an adapter expires. - - This method must be overridden by inheriting classes to handle - adapter expiration logic. The base implementation raises NotImplementedError. - - Args: - adapter_name: Name of the expired adapter. - - Raises: - NotImplementedError: If not overridden by inheriting class. - """ - raise NotImplementedError(f'_on_adapter_expired must be implemented by {self.__class__.__name__}') + """Hook method called when an adapter expires.""" + ... @staticmethod def get_adapter_name(adapter_name: str) -> str: diff --git a/src/twinkle/server/utils/session_resource/base.py b/src/twinkle/server/utils/session_resource/base.py index 83c26189..4411f12b 100644 --- a/src/twinkle/server/utils/session_resource/base.py +++ b/src/twinkle/server/utils/session_resource/base.py @@ -9,19 +9,19 @@ import asyncio import time -from abc import abstractmethod +from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from twinkle.server.state import ServerState -from twinkle.server.exceptions import ResourceNotFoundError +from twinkle.server.exceptions import RequestRejectedError, ResourceNotFoundError from twinkle.utils.logger import get_logger logger = get_logger() -class SessionResourceMixin: +class SessionResourceMixin(ABC): """Base mixin for managing session-bound resources with automatic expiration. This mixin tracks resources and automatically expires them when their @@ -63,37 +63,47 @@ def _init_resource_manager( self._resource_max_lifetime = resource_max_lifetime # Resource lifecycle tracking - # Dict mapping resource_id -> - # {'token': str, 'session_id': str, 'created_at': float, 'state': dict, 'expiring': bool} + # Dict mapping resource_id -> lifecycle metadata, including the last time + # the shared state backend confirmed that the owning session was alive. self._resource_records: dict[str, dict[str, Any]] = {} # Countdown task self._resource_countdown_running = False self._countdown_task: asyncio.Task | None = None - async def _is_session_alive(self, session_id: str) -> bool: - """Check if a session is still alive via state proxy. + async def _is_session_alive(self, session_id: str, resource_id: str, record: dict[str, Any]) -> bool: + """Check session liveness with a bounded fail-open window. - Args: - session_id: Session ID to check - - Returns: - True if session is alive, False if expired or not found + ``record`` is the caller-held lifecycle dict (the same object stored in + ``_resource_records``), passed in so a concurrent ``unregister_resource`` + cannot turn a by-key lookup here into a ``KeyError`` mid-sweep. """ if not session_id: - return True # No session association means always alive + raise RuntimeError(f'registered {self._resource_type} {resource_id} has no session_id') try: last_heartbeat = await self.state.get_session_last_heartbeat(session_id) - except Exception as e: - logger.warning(f'[{self._resource_type}Manager] Failed to check session liveness: {e}') - return True # Assume alive on error + except Exception as exc: + elapsed = time.time() - record['last_liveness_confirmed_at'] + logger.warning( + '[%sManager] Session liveness probe failed for %s; bounded fail-open ' + 'elapsed=%.3fs timeout=%.3fs error=%r', + self._resource_type, + resource_id, + elapsed, + self._resource_timeout, + exc, + ) + return elapsed < self._resource_timeout if last_heartbeat is None: - return False # Session doesn't exist + return False - # Check if session has timed out - return (time.time() - last_heartbeat) < self._resource_timeout + now = time.time() + alive = (now - last_heartbeat) < self._resource_timeout + if alive: + record['last_liveness_confirmed_at'] = now + return alive def _validate_registration(self, resource_id: str, token: str, session_id: str) -> None: """Validate before registering a resource. Override for custom validation. @@ -104,11 +114,11 @@ def _validate_registration(self, resource_id: str, token: str, session_id: str) session_id: Session ID Raises: - ValueError: If validation fails - RuntimeError: If resource limit is reached + RequestRejectedError: If the required session ID is absent. """ if not session_id: - raise ValueError(f'session_id must be provided when registering {self._resource_type} {resource_id}') + raise RequestRejectedError( + f'session_id must be provided when registering {self._resource_type} {resource_id}') def _create_resource_record(self, token: str, session_id: str) -> dict[str, Any]: """Create a new resource record. Override to add custom fields. @@ -120,12 +130,14 @@ def _create_resource_record(self, token: str, session_id: str) -> dict[str, Any] Returns: Resource record dict """ + now = time.time() return { 'token': token, 'session_id': session_id, - 'created_at': time.time(), + 'created_at': now, 'state': {}, 'expiring': False, + 'last_liveness_confirmed_at': now, } def register_resource(self, resource_id: str, token: str, session_id: str) -> None: @@ -137,8 +149,7 @@ def register_resource(self, resource_id: str, token: str, session_id: str) -> No session_id: Session ID to associate with this resource. Raises: - ValueError: If session_id is None or empty. - RuntimeError: If custom validation fails (e.g., limit reached). + RequestRejectedError: If session_id is None or empty. """ self._validate_registration(resource_id, token, session_id) @@ -239,6 +250,14 @@ def assert_resource_exists(self, resource_id: str) -> None: if not (resource_id and info is not None and not info.get('expiring')): raise ResourceNotFoundError(f'{self._resource_type} {resource_id} not found') + async def _on_resource_liveness_confirmed(self, resource_id: str) -> bool: + """Refresh any resource-specific lease after a successful session probe. + + Returns ``False`` when a resource-specific lease has already been lost and + the local resource must be expired to avoid running outside its quota. + """ + return True + @abstractmethod async def _on_resource_expired(self, resource_id: str) -> None: """Hook method called when a resource expires. @@ -287,12 +306,9 @@ async def _resource_countdown_loop(self) -> None: expired_resources.append((resource_id, token, session_id)) continue - try: - session_alive = await self._is_session_alive(session_id) - except Exception as e: - logger.warning(f'[{self._resource_type}Manager] Failed to check session liveness ' - f'for {resource_id}: {type(e).__name__}: {e}') - continue + session_alive = await self._is_session_alive(session_id, resource_id, info) + if session_alive: + session_alive = await self._on_resource_liveness_confirmed(resource_id) session_expired = not session_alive logger.debug(f'[{self._resource_type}Manager] {self._resource_type} {resource_id} session check ' f'(session_id={session_id}, session_alive={not session_expired})') diff --git a/src/twinkle/server/utils/session_resource/processor.py b/src/twinkle/server/utils/session_resource/processor.py index 2b75270b..69a6192e 100644 --- a/src/twinkle/server/utils/session_resource/processor.py +++ b/src/twinkle/server/utils/session_resource/processor.py @@ -7,8 +7,7 @@ """ from __future__ import annotations -import time -from typing import Any +from abc import abstractmethod from twinkle.utils.logger import get_logger from .base import SessionResourceMixin @@ -51,34 +50,30 @@ def _init_processor_manager( resource_max_lifetime=None, # No max lifetime for processors ) self._per_token_processor_limit = per_token_processor_limit - - def _validate_registration(self, resource_id: str, token: str, session_id: str) -> None: - """Validate before registering a processor. Checks per-token limit. - - Args: - resource_id: Processor identifier - token: User token - session_id: Session ID - - Raises: - ValueError: If session_id is empty. - RuntimeError: If per-token limit is reached. - """ - super()._validate_registration(resource_id, token, session_id) - - current_count = sum(1 for info in self._resource_records.values() if info.get('token') == token) - if current_count >= self._per_token_processor_limit: - raise RuntimeError(f'Per-user processor limit ({self._per_token_processor_limit}) reached ' - f'for token {token[:8]}...') - - def _create_resource_record(self, token: str, session_id: str) -> dict[str, Any]: - """Create a new processor record without state field.""" - return { - 'token': token, - 'session_id': session_id, - 'created_at': time.time(), - 'expiring': False, - } + # The countdown runs every 10 seconds. A 30-second lease tolerates two + # missed renewals while still bounding stale reservations after a crash. + self._processor_quota_lease_seconds = 30.0 + + async def _on_resource_liveness_confirmed(self, resource_id: str) -> bool: + """Renew this processor's shared quota lease after a healthy probe.""" + info = self._resource_records.get(resource_id) + if info is None: + return False + try: + renewed = await self.state.renew_processor_quota( + info['token'], + resource_id, + lease_seconds=self._processor_quota_lease_seconds, + ) + except Exception as exc: + # Keep the local processor during a transient backend outage. Once the + # backend recovers, a lost/expired reservation returns False and the + # countdown loop removes the unaccounted local resource. + logger.warning('[ProcessorManager] Failed to renew quota lease for %s: %r', resource_id, exc) + return True + if not renewed: + logger.warning('[ProcessorManager] Quota lease for %s was lost; expiring local processor', resource_id) + return renewed async def _on_resource_expired(self, resource_id: str) -> None: """Base-class expiry hook; forwards to the domain hook ``_on_processor_expired``. @@ -91,12 +86,7 @@ async def _on_resource_expired(self, resource_id: str) -> None: """ await self._on_processor_expired(resource_id) + @abstractmethod async def _on_processor_expired(self, processor_id: str) -> None: - """Hook called when a processor's session expires. - - Must be overridden by inheriting classes. - - Raises: - NotImplementedError: If not overridden. - """ - raise NotImplementedError(f'_on_processor_expired must be implemented by {self.__class__.__name__}') + """Hook called when a processor's session expires.""" + ... diff --git a/src/twinkle/server/utils/task_errors.py b/src/twinkle/server/utils/task_errors.py index cfa85257..2ad62111 100644 --- a/src/twinkle/server/utils/task_errors.py +++ b/src/twinkle/server/utils/task_errors.py @@ -1,12 +1,7 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Construction and backward-compatible reading of failure payloads. - -``ErrorPayload`` is the single representation of a failure both on the wire and in -state. This module owns the two entry points that produce/repair it. -""" +"""Helpers for constructing protocol-layer ``ErrorPayload`` values.""" from __future__ import annotations -from collections.abc import Mapping from typing import Any from twinkle_client.types.errors import ErrorCategory, ErrorPayload @@ -16,7 +11,7 @@ _TRUNCATION_MARKER = '...[traceback truncated, tail kept]...\n' -def _trim_traceback(text: str) -> str: +def trim_traceback(text: str) -> str: """Keep the tail of an over-long traceback (innermost frames are densest).""" if len(text) <= _TRACEBACK_MAX: return text @@ -31,15 +26,12 @@ def task_error_payload( error_code: int = 500, category: ErrorCategory | str = ErrorCategory.Server, traceback_text: str | None = None, + details: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: - """Build an ``ErrorPayload`` and return it as a JSON-safe dict for storage. - - Traceback splitting and length trimming happen here so over-long text is never - written to State_Backend. A ``user`` category carries no traceback. - """ + """Build a JSON-safe wire payload for direct or streaming responses.""" if isinstance(category, str): category = ErrorCategory(category.lower()) - tb = _trim_traceback(traceback_text) if category is ErrorCategory.Server and traceback_text else None + tb = trim_traceback(traceback_text) if category is ErrorCategory.Server and traceback_text else None lines = str(error).splitlines() summary = (lines[0] if lines else '')[:_ERROR_MAX] payload = ErrorPayload( @@ -48,26 +40,6 @@ def task_error_payload( error_code=error_code, request_id=request_id, traceback=tb, + details=details, ) return payload.model_dump(mode='json', exclude_none=True) - - -def error_payload_from_stored(stored: Any, *, request_id: str) -> ErrorPayload: - """Build an ``ErrorPayload`` from whatever is sitting in ``FutureRecord.result``. - - Records written before this spec have only ``{error, category}``. Missing - ``error_code`` / ``request_id`` / ``category`` are backfilled with ``500`` / - the caller-supplied value / ``Unknown`` so a rolling upgrade never raises - ``pydantic.ValidationError``. - """ - if isinstance(stored, Mapping): - data = dict(stored) - else: - data = {'error': 'Unknown error' if stored is None else str(stored)} - data.setdefault('category', ErrorCategory.Unknown) - data.setdefault('error_code', 500) - data.setdefault('request_id', request_id) - category = str(data['category']).lower() - if category != ErrorCategory.Server.value: - data.pop('traceback', None) - return ErrorPayload.model_validate(data) diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/utils/task_queue/mixin.py index eb02f883..fea48fbb 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/utils/task_queue/mixin.py @@ -20,8 +20,8 @@ from twinkle.server.exceptions import BatchSizeError, ConfigError, InputTokensExceededError, RateLimitExceededError from twinkle.server.lifecycle.envelope import envelope_from_record from twinkle.server.lifecycle.poll_config import long_poll_window +from twinkle.server.state.models import FutureFailureRecord from twinkle.server.telemetry.middleware import get_task_metrics -from twinkle.server.utils.task_errors import task_error_payload from twinkle.utils.logger import get_logger from twinkle_client.types.lifecycle import TERMINAL_STATUSES, TaskEnvelope from .config import TaskQueueConfig @@ -475,17 +475,17 @@ async def _run() -> None: ) logger.info(f'[TaskQueue] Background task {request_id} completed, type={task_type or "unknown"}') except Exception as exc: - error_payload = task_error_payload( - f'{type(exc).__name__}: {exc}', - request_id=request_id, - error_code=500, - traceback_text=traceback.format_exc(), + failure = FutureFailureRecord( + reason_code='internal_error', + message=f'{type(exc).__name__}: {exc}'[:1024], + attribution='server', + diagnostic=traceback.format_exc(), ) await self.state.store_future_status( request_id, TaskStatus.FAILED.value, model_id, - result=error_payload, + failure=failure, queue_state=QueueState.ACTIVE.value, ) logger.error(f'[TaskQueue] Background task {request_id} FAILED, type={task_type or "unknown"}:\n' diff --git a/src/twinkle/server/utils/task_queue/worker.py b/src/twinkle/server/utils/task_queue/worker.py index 606f5003..f163a966 100644 --- a/src/twinkle/server/utils/task_queue/worker.py +++ b/src/twinkle/server/utils/task_queue/worker.py @@ -14,10 +14,14 @@ from collections import deque from typing import TYPE_CHECKING, Any, Callable, Deque -from twinkle.server.exceptions import TwinkleServerError +from twinkle.server.exceptions import (BatchSizeError, EndpointUnavailableError, FullModeBusyError, + InputTokensExceededError, RateLimitExceededError, RequestRejectedError, + ResourceNotFoundError, ResourceQuotaExceededError, StateBackendError, + TrainModeMismatchError, TwinkleServerError) +from twinkle.server.state.backend.base import ConcurrencyError +from twinkle.server.state.models import FutureFailureRecord from twinkle.server.telemetry.correlation import MODEL_ID, TOKEN_ID from twinkle.server.telemetry.tracing import traced_operation -from twinkle.server.utils.task_errors import task_error_payload from twinkle.utils.logger import get_logger from twinkle_client.types.errors import ErrorCategory from .config import TaskQueueConfig @@ -29,6 +33,28 @@ logger = get_logger() + +def _reason_code_for_server_error(exc: TwinkleServerError) -> str: + """Map typed execution failures to protocol-independent reason codes.""" + mappings: tuple[tuple[type[TwinkleServerError], str], ...] = ( + (TrainModeMismatchError, 'invalid_request'), + (ResourceNotFoundError, 'resource_not_found'), + (FullModeBusyError, 'full_mode_busy'), + (InputTokensExceededError, 'input_tokens_exceeded'), + (BatchSizeError, 'batch_size_invalid'), + (RateLimitExceededError, 'rate_limit_exceeded'), + (ResourceQuotaExceededError, 'resource_quota_exceeded'), + (EndpointUnavailableError, 'endpoint_unavailable'), + (ConcurrencyError, 'state_contention'), + (StateBackendError, 'backend_gate_unavailable'), + (RequestRejectedError, 'request_rejected'), + ) + for error_type, reason_code in mappings: + if isinstance(exc, error_type): + return reason_code + return 'internal_error' + + # Ray_Get_Timeout is classified the same as asyncio.TimeoutError: 504/Server. try: from ray.exceptions import GetTimeoutError as _RayGetTimeout @@ -157,27 +183,22 @@ async def _store_task_failed( queue_state: str, queue_state_reason: str | None = None, *, - error_code: int = 500, - category: ErrorCategory = ErrorCategory.Server, + reason_code: str = 'internal_error', + attribution: str = 'server', traceback_text: str | None = None, ) -> None: - """Store FAILED status with a standardised ``ErrorPayload``. - - The future record is the single delivery channel: a failed task is written - unconditionally so both the Inline_Fast_Path peek and the Retrieve_Endpoint - observe the same terminal record. - """ + """Store FAILED status with protocol-independent failure details.""" + failure = FutureFailureRecord( + reason_code=reason_code, + message=(error.splitlines() or [''])[0][:1024], + attribution='user' if attribution == 'user' else 'server', + diagnostic=traceback_text, + ) await self._state.store_future_status( task.request_id, TaskStatus.FAILED.value, task.model_id, - result=task_error_payload( - error, - request_id=task.request_id, - error_code=error_code, - category=category, - traceback_text=traceback_text, - ), + failure=failure, queue_state=queue_state, queue_state_reason=queue_state_reason, ) @@ -279,7 +300,8 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue logger.error(f'[ComputeWorker] Task {task.request_id} TIMEOUT after {exec_time:.2f}s, ' f'type={task_type}, queue_key={queue_key}') # asyncio.TimeoutError and Ray_Get_Timeout are 504/Server. - await self._store_task_failed(task, error, QueueState.ACTIVE.value, error_code=504) + await self._store_task_failed( + task, error, QueueState.ACTIVE.value, reason_code='execution_timeout') # Probe actor liveness after a timeout so an operator learns the replica's # state without waiting for a second request to also time out. if self._on_backend_timeout is not None: @@ -294,8 +316,8 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue task, f'{type(exc).__name__}: {exc}', QueueState.UNKNOWN.value, - error_code=400, - category=ErrorCategory.User, + reason_code='request_rejected', + attribution='user', ) except BackendBusyError as exc: task_status = 'failed' @@ -304,7 +326,8 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue logger.error(f'[ComputeWorker] Task {task.request_id} REFUSED (admission gate held) after ' f'{exec_time:.2f}s, type={task_type}, queue_key={queue_key}') # Gate held by a leaked timed-out call -> 503/Server. - await self._store_task_failed(task, error, QueueState.ACTIVE.value, error_code=503) + await self._store_task_failed( + task, error, QueueState.ACTIVE.value, reason_code='backend_gate_unavailable') except TwinkleServerError as exc: # A typed server error carries its own status + category (e.g. # ResourceNotFoundError = 404/User from a deferred @@ -321,8 +344,8 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue task, f'{type(exc).__name__}: {exc}', QueueState.ACTIVE.value, - error_code=exc.error_code, - category=exc.category, + reason_code=_reason_code_for_server_error(exc), + attribution=exc.category.value, traceback_text=traceback.format_exc() if is_server else None, ) except Exception as exc: @@ -334,7 +357,12 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue logger.error(f'[ComputeWorker] Task {task.request_id} FAILED after {exec_time:.2f}s, ' f'type={task_type}:\n{traceback.format_exc(limit=3)}') await self._store_task_failed( - task, error, QueueState.ACTIVE.value, error_code=500, traceback_text=traceback.format_exc()) + task, + error, + QueueState.ACTIVE.value, + reason_code='internal_error', + traceback_text=traceback.format_exc(), + ) finally: q.task_done() self._record_execution_time(task_type, exec_time) diff --git a/src/twinkle/server/validation/__init__.py b/src/twinkle/server/validation/__init__.py index 5a9afdc4..6ff764dd 100644 --- a/src/twinkle/server/validation/__init__.py +++ b/src/twinkle/server/validation/__init__.py @@ -20,12 +20,11 @@ :mod:`.errors` is the other half of the story: it gives FastAPI's own body-validation failures the same wire shape as these, so a caller sees one error format. """ -from .backend_compat import BackendCapability, EndpointUnavailableError, assert_request_supported, resolve_backend +from .backend_compat import BackendCapability, assert_request_supported, resolve_backend from .errors import register_validation_error_handler __all__ = [ 'BackendCapability', - 'EndpointUnavailableError', 'assert_request_supported', 'register_validation_error_handler', 'resolve_backend', diff --git a/src/twinkle/server/validation/backend_compat.py b/src/twinkle/server/validation/backend_compat.py index 6587ab4b..82f4ab9e 100644 --- a/src/twinkle/server/validation/backend_compat.py +++ b/src/twinkle/server/validation/backend_compat.py @@ -28,9 +28,8 @@ from enum import StrEnum from typing import Any, Optional -from twinkle.server.exceptions import RequestRejectedError, TwinkleServerError +from twinkle.server.exceptions import EndpointUnavailableError, RequestRejectedError from twinkle_client.types.base import FieldRole, fields_with_role, read_backend_only -from twinkle_client.types.errors import ErrorCategory class BackendCapability(StrEnum): @@ -56,18 +55,6 @@ class BackendCapability(StrEnum): _ALTERNATIVES = 'use `forward_backward` (training) or `forward_only` (inference) instead' -class EndpointUnavailableError(TwinkleServerError): - """The endpoint is not implemented by this deployment's backend. - - 501, not 4xx: the request is well-formed and the caller is not at fault -- this - deployment simply cannot serve it. ``Server`` category, and no traceback, because - the answer is a deployment fact rather than a crash. - """ - - error_code = 501 - category = ErrorCategory.Server - - def resolve_backend(service: Any) -> str | None: """This deployment's declared backend, or ``None`` when it has no backend concept. diff --git a/src/twinkle_client/exceptions.py b/src/twinkle_client/exceptions.py index 2e61c476..e9a79c63 100644 --- a/src/twinkle_client/exceptions.py +++ b/src/twinkle_client/exceptions.py @@ -18,6 +18,8 @@ import requests from typing import Any, Optional +from twinkle_client.types.errors import ErrorCategory + class TwinkleClientValidationError(ValueError): """A caller argument could not be placed in the request model, in-process. @@ -37,7 +39,8 @@ class TwinkleHTTPError(requests.HTTPError): Inherits ``requests.HTTPError`` so callers already catching that keep working. ``status_code`` is the HTTP status; ``error_code`` / ``category`` come from the - server's structured error body when present (else ``None`` / ``'Unknown'``). + server's structured error body when present. ``category`` always uses the + lowercase :class:`ErrorCategory` wire value. """ def __init__( @@ -45,8 +48,10 @@ def __init__( *args: Any, status_code: int | None = None, error_code: int | None = None, - category: str = 'Unknown', + category: str = ErrorCategory.Unknown.value, request_id: str | None = None, + details: list[dict[str, Any]] | None = None, + traceback: str | None = None, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) @@ -54,6 +59,8 @@ def __init__( self.error_code = error_code self.category = category self.request_id = request_id + self.details = details + self.traceback = traceback class TaskFailedError(Exception): diff --git a/src/twinkle_client/http/client.py b/src/twinkle_client/http/client.py index a867bff8..77719c78 100644 --- a/src/twinkle_client/http/client.py +++ b/src/twinkle_client/http/client.py @@ -8,6 +8,7 @@ from typing import Any from twinkle_client.exceptions import TwinkleClientValidationError, TwinkleHTTPError +from twinkle_client.types.errors import ErrorCategory, ErrorPayload from .context import ClientContext, capture_transport from .headers import build_routing_headers @@ -51,13 +52,29 @@ def _handle_response(response: requests.Response) -> requests.Response: body = response.json() except Exception: body = None + payload: ErrorPayload | None = None if isinstance(body, dict): - category = body.get('category', 'Unknown') - error_code = body.get('error_code') - request_id = body.get('request_id') - summary = body.get('error') or body.get('detail') or response.text + try: + payload = ErrorPayload.model_validate(body) + except Exception: + payload = None + + if payload is not None: + summary = payload.error or response.text + category = payload.category.value + error_code = payload.error_code + request_id = payload.request_id + details = payload.details + traceback_text = payload.traceback + elif isinstance(body, dict): + summary = body.get('detail') or response.text + category = ErrorCategory.Unknown.value + error_code = request_id = details = traceback_text = None else: - category, error_code, request_id, summary = 'Unknown', None, None, response.text + summary = response.text + category = ErrorCategory.Unknown.value + error_code = request_id = details = traceback_text = None + message = f'{response.status_code} Error for url: {response.url}\nServer detail:\n{summary}' raise TwinkleHTTPError( message, @@ -66,6 +83,8 @@ def _handle_response(response: requests.Response) -> requests.Response: error_code=error_code, category=category, request_id=request_id, + details=details, + traceback=traceback_text, ) diff --git a/tests/server/config/test_server_config.py b/tests/server/config/test_server_config.py index 9c50cb2a..b5bacf82 100644 --- a/tests/server/config/test_server_config.py +++ b/tests/server/config/test_server_config.py @@ -9,6 +9,8 @@ """ from __future__ import annotations +import re + import pytest import yaml from hypothesis import given, settings @@ -19,6 +21,7 @@ from twinkle.server.config import ApplicationSpec, ServerConfig from twinkle.server.exceptions import ConfigParseError from twinkle.server.launcher import ServerLauncher +from twinkle.server.sampler.app import SAMPLER_SELECTOR, build_sampler_app # ---------- minimal valid config strategy ---------------------------------- # @@ -115,6 +118,26 @@ def test_nested_field_constraint_violation_named(bad_max_input_tokens: int) -> N assert any('max_input_tokens' in err['loc'] for err in errors) +def test_torch_sampler_is_rejected_during_config_validation() -> None: + with pytest.raises(ValidationError): + ApplicationSpec.model_validate({ + 'name': 'sampler', + 'import_path': 'sampler', + 'args': { + 'model_id': 'm', + 'device_group': {}, + 'device_mesh': {}, + 'sampler_type': 'torch', + }, + }) + + +def test_sampler_docstring_values_match_selector() -> None: + doc = build_sampler_app.__doc__ or '' + line = next(line for line in doc.splitlines() if 'sampler_type:' in line) + assert set(re.findall(r'``(\w+)``', line)) == set(SAMPLER_SELECTOR.builders) + + # ---------- round-trip fidelity ----------------------------------------- # diff --git a/tests/server/conftest.py b/tests/server/conftest.py index fea2e62c..de4644de 100644 --- a/tests/server/conftest.py +++ b/tests/server/conftest.py @@ -49,7 +49,7 @@ def _reset_canonical_state_actor(): """Clear the canonical state actor's store before each test function. Hypothesis property tests reuse the function scope across examples and - so should call ``backend.close()`` themselves to reset between examples. + so should call the actor's explicit ``flush_all()`` test hook themselves. """ import ray @@ -62,7 +62,7 @@ def _reset_canonical_state_actor(): actor = None if actor is not None: try: - ray.get(actor.close.remote()) + ray.get(actor.flush_all.remote()) except Exception: pass yield diff --git a/tests/server/contract/test_error_wire.py b/tests/server/contract/test_error_wire.py index df6be289..79ca7785 100644 --- a/tests/server/contract/test_error_wire.py +++ b/tests/server/contract/test_error_wire.py @@ -12,11 +12,10 @@ class _State: async def get_future(self, request_id: str): return { 'status': 'failed', - 'result': { - 'error': 'backend timed out', - 'category': 'server', - 'error_code': 504, - 'request_id': request_id, + 'failure': { + 'reason_code': 'execution_timeout', + 'message': 'backend timed out', + 'attribution': 'server', }, } diff --git a/tests/server/lifecycle/test_envelope.py b/tests/server/lifecycle/test_envelope.py index f7ff6321..2c2216f7 100644 --- a/tests/server/lifecycle/test_envelope.py +++ b/tests/server/lifecycle/test_envelope.py @@ -2,9 +2,24 @@ """Unit tests for the single FutureRecord -> TaskEnvelope mapping point (T1.2).""" from __future__ import annotations +import pytest +from pydantic import ValidationError + from twinkle.server.lifecycle.envelope import envelope_from_record +def test_wire_maps_cover_exactly_the_canonical_reason_codes(): + """The Twinkle and Tinker failure maps must both stay in lockstep with the + canonical reason-code set, so a newly added domain reason cannot silently + fall back to 500 on one protocol face.""" + from twinkle.server.gateway.tinker_handlers import _TINKER_FAILURE_WIRE + from twinkle.server.lifecycle.envelope import _FAILURE_WIRE + from twinkle.server.state.models import FAILURE_REASON_CODES + + assert set(_FAILURE_WIRE) == FAILURE_REASON_CODES + assert set(_TINKER_FAILURE_WIRE) == FAILURE_REASON_CODES + + def test_completed_with_none_result_is_a_success_not_a_failure(): """R1#4 / Property 4: `completed` + `result is None` is a valid success.""" env = envelope_from_record('req-1', {'status': 'completed', 'result': None}) @@ -19,20 +34,31 @@ def test_completed_carries_result_and_no_error(): assert env.error is None -def test_legacy_two_field_failure_payload_is_backfilled_not_strict_validated(): - """R2#8 / Property 5: a pre-spec {error, category} payload maps to a legal envelope. - - It must go through error_payload_from_stored (missing error_code/request_id are - backfilled), never a strict ErrorPayload.model_validate that would raise. - """ - env = envelope_from_record('req-9', {'status': 'failed', 'result': {'error': 'boom', 'category': 'server'}}) +def test_domain_failure_maps_to_twinkle_error_payload(): + env = envelope_from_record( + 'req-9', { + 'status': 'failed', + 'failure': { + 'reason_code': 'execution_timeout', + 'message': 'backend timed out', + 'attribution': 'server', + 'diagnostic': 'full traceback', + }, + }) assert env.status == 'failed' assert env.result is None assert env.error is not None - assert env.error.error == 'boom' + assert env.error.error == 'backend timed out' assert env.error.category.value == 'server' - assert env.error.error_code == 500 # backfilled - assert env.error.request_id == 'req-9' # backfilled from the argument + assert env.error.error_code == 504 + assert env.error.request_id == 'req-9' + assert env.error.traceback == 'full traceback' + + +def test_legacy_failure_in_result_is_not_accepted(): + with pytest.raises(ValidationError): + envelope_from_record( + 'req-old', {'status': 'failed', 'result': {'error': 'boom', 'category': 'server'}}) def test_non_terminal_record_carries_queue_state_and_no_payload(): diff --git a/tests/server/lifecycle/test_retrieve_endpoint.py b/tests/server/lifecycle/test_retrieve_endpoint.py index f2f78033..fcc0d6e7 100644 --- a/tests/server/lifecycle/test_retrieve_endpoint.py +++ b/tests/server/lifecycle/test_retrieve_endpoint.py @@ -11,6 +11,8 @@ from fastapi import FastAPI from fastapi.testclient import TestClient +from twinkle.server.deployment import twinkle_server_error_handler +from twinkle.server.exceptions import TwinkleServerError from twinkle.server.gateway.twinkle_handlers import _register_twinkle_routes @@ -32,6 +34,7 @@ def __init__(self, record): def _client(record) -> TestClient: app = FastAPI() + app.add_exception_handler(TwinkleServerError, twinkle_server_error_handler) _register_twinkle_routes(app, lambda: _Gateway(record)) return TestClient(app) @@ -47,9 +50,15 @@ def test_completed_with_null_result_returns_200_and_null(monkeypatch): assert body['error'] is None -def test_legacy_two_field_failure_returns_200_and_valid_envelope(): - """Property 5 / R8#5: a {error, category} record is 200 with a legal envelope.""" - client = _client({'status': 'failed', 'result': {'error': 'boom', 'category': 'server'}}) +def test_domain_failure_returns_200_and_valid_envelope(): + client = _client({ + 'status': 'failed', + 'failure': { + 'reason_code': 'internal_error', + 'message': 'boom', + 'attribution': 'server', + }, + }) resp = client.post('/twinkle/retrieve_future', json={'request_id': 'req-2'}) assert resp.status_code == 200 body = resp.json() @@ -69,7 +78,9 @@ def test_always_missing_record_404s_only_after_the_full_window(monkeypatch): resp = client.post('/twinkle/retrieve_future', json={'request_id': 'ghost'}) waited = time.monotonic() - start assert resp.status_code == 404 - assert 'ghost' in resp.json()['detail'] + assert 'ghost' in resp.json()['error'] + assert resp.json()['category'] == 'user' + assert resp.json()['error_code'] == 404 # It must fold the missing record into the wait loop, not short-circuit. assert waited >= 0.3 diff --git a/tests/server/lifecycle/test_tinker_retrieve_regression.py b/tests/server/lifecycle/test_tinker_retrieve_regression.py index ab8d07b3..257c283a 100644 --- a/tests/server/lifecycle/test_tinker_retrieve_regression.py +++ b/tests/server/lifecycle/test_tinker_retrieve_regression.py @@ -44,8 +44,15 @@ def test_try_again_shape_for_non_terminal(monkeypatch): def test_error_category_shape_for_failed(): - resp = _client({'status': 'failed', 'result': {'error': 'boom', 'category': 'server'}}).post( - '/retrieve_future', json={'request_id': 'r'}) + record = { + 'status': 'failed', + 'failure': { + 'reason_code': 'internal_error', + 'message': 'boom', + 'attribution': 'server', + }, + } + resp = _client(record).post('/retrieve_future', json={'request_id': 'r'}) assert resp.status_code == 200 body = resp.json() assert body['error'] == 'boom' diff --git a/tests/server/session_resource/test_contract.py b/tests/server/session_resource/test_contract.py new file mode 100644 index 00000000..b60f70f4 --- /dev/null +++ b/tests/server/session_resource/test_contract.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import asyncio +from contextlib import suppress +from unittest import mock + +import pytest + +from twinkle.server.exceptions import RequestRejectedError +from twinkle.server.utils.session_resource.adapter import AdapterManagerMixin +from twinkle.server.utils.session_resource.base import SessionResourceMixin +from twinkle.server.utils.session_resource.processor import ProcessorManagerMixin + + +class _State: + + def __init__(self, outcomes: list[object] | None = None) -> None: + self.outcomes = list(outcomes or []) + + async def get_session_last_heartbeat(self, session_id: str) -> float | None: + outcome = self.outcomes.pop(0) + if isinstance(outcome, Exception): + raise outcome + return outcome # type: ignore[return-value] + + +class _ResourceManager(SessionResourceMixin): + + def __init__(self, state: _State, timeout: float = 10.0) -> None: + self.state = state + self.expired: list[str] = [] + self._init_resource_manager(resource_timeout=timeout) + + async def _on_resource_expired(self, resource_id: str) -> None: + self.expired.append(resource_id) + + +class _MissingBaseHook(SessionResourceMixin): + pass + + +class _MissingAdapterHook(AdapterManagerMixin): + pass + + +class _MissingProcessorHook(ProcessorManagerMixin): + pass + + +def test_all_resource_expiry_hooks_are_abstract() -> None: + for cls in (_MissingBaseHook, _MissingAdapterHook, _MissingProcessorHook): + with pytest.raises(TypeError): + cls() + + +def test_registration_requires_session_id() -> None: + manager = _ResourceManager(_State()) + with pytest.raises(RequestRejectedError): + manager.register_resource('r1', 'token', '') + + +@pytest.mark.asyncio +async def test_liveness_failure_has_hard_upper_bound_and_recovery_refreshes() -> None: + state = _State([RuntimeError('down'), 109.0, RuntimeError('down'), RuntimeError('down')]) + manager = _ResourceManager(state, timeout=10.0) + with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=100.0): + manager.register_resource('r1', 'token', 'session') + record = manager.get_resource_info('r1') + + with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=105.0): + assert await manager._is_session_alive('session', 'r1', record) is True + with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=110.0): + assert await manager._is_session_alive('session', 'r1', record) is True + assert manager.get_resource_info('r1')['last_liveness_confirmed_at'] == 110.0 + with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=115.0): + assert await manager._is_session_alive('session', 'r1', record) is True + with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=120.0): + assert await manager._is_session_alive('session', 'r1', record) is False + + +@pytest.mark.asyncio +async def test_countdown_restart_preserves_confirmation_time() -> None: + manager = _ResourceManager(_State([100.0])) + with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=100.0): + manager.register_resource('r1', 'token', 'session') + confirmed_at = manager.get_resource_info('r1')['last_liveness_confirmed_at'] + + manager._ensure_countdown_started() + first_task = manager._countdown_task + manager.stop_resource_countdown() + with suppress(asyncio.CancelledError): + await first_task + + manager._ensure_countdown_started() + second_task = manager._countdown_task + assert second_task is not first_task + assert manager.get_resource_info('r1')['last_liveness_confirmed_at'] == confirmed_at + manager.stop_resource_countdown() + with suppress(asyncio.CancelledError): + await second_task + + +def test_worker_restart_does_not_restore_local_resources() -> None: + old = _ResourceManager(_State()) + old.register_resource('r1', 'token', 'session') + restarted = _ResourceManager(_State()) + assert restarted.get_resource_info('r1') is None diff --git a/tests/server/state/fake_backend.py b/tests/server/state/fake_backend.py index 247cd662..f59a75ae 100644 --- a/tests/server/state/fake_backend.py +++ b/tests/server/state/fake_backend.py @@ -72,5 +72,11 @@ async def update_atomic( await self.set(key, updated, ttl) return updated - async def close(self) -> None: + async def flush_all(self) -> None: self._store.clear() + + async def close(self) -> None: + pass + + async def health_check(self) -> bool: + return True diff --git a/tests/server/state/test_error_payload.py b/tests/server/state/test_error_payload.py index eb224f73..aabc1e2a 100644 --- a/tests/server/state/test_error_payload.py +++ b/tests/server/state/test_error_payload.py @@ -1,29 +1,14 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Tests for ErrorPayload construction, backfill, and tinker-SDK wire compat. - -Spec: T2.4 / R9#7 / R8#5. -""" +"""Tests for direct/streaming ErrorPayload construction and Tinker parsing.""" from __future__ import annotations import pytest from pydantic import ValidationError -from twinkle.server.utils.task_errors import error_payload_from_stored, task_error_payload +from twinkle.server.utils.task_errors import task_error_payload from twinkle_client.types.errors import ErrorCategory, ErrorPayload -def test_two_field_legacy_backfills_error_code_and_request_id(): - """A pre-spec {error, category} payload backfills to 500 + passed request_id.""" - stored = {'error': 'boom', 'category': 'Server'} - - payload = error_payload_from_stored(stored, request_id='req_42') - - assert isinstance(payload, ErrorPayload) - assert payload.error_code == 500 - assert payload.request_id == 'req_42' - assert payload.error == 'boom' - - def test_overlong_traceback_is_trimmed_tail_kept_with_marker(): long_tb = 'X' * 10 + ('line\n' * 40000) # well over 65536 chars assert len(long_tb) > 65536 @@ -72,12 +57,6 @@ def test_tinker_sdk_parses_six_field_like_two_field(): assert parsed_six.category == parsed_two.category -def test_legacy_title_case_category_is_normalized(): - payload = error_payload_from_stored({'error': 'boom', 'category': 'Server'}, request_id='req_10') - assert payload.category is ErrorCategory.Server - assert payload.category.value == 'server' - - @pytest.mark.parametrize('category', [ErrorCategory.User, ErrorCategory.Unknown]) def test_non_server_traceback_is_rejected(category): with pytest.raises(ValidationError): @@ -88,10 +67,3 @@ def test_non_server_traceback_is_rejected(category): request_id='req_11', traceback='server stack', ) - - -def test_legacy_unknown_traceback_is_removed(): - payload = error_payload_from_stored( - {'error': 'legacy', 'category': 'Unknown', 'traceback': 'old stack'}, request_id='req_12') - assert payload.category is ErrorCategory.Unknown - assert payload.traceback is None diff --git a/tests/server/state/test_future_lifecycle.py b/tests/server/state/test_future_lifecycle.py index d423dcfe..064bed11 100644 --- a/tests/server/state/test_future_lifecycle.py +++ b/tests/server/state/test_future_lifecycle.py @@ -17,6 +17,7 @@ from twinkle.server.state.backend.base import StateBackend from twinkle.server.state.future_manager import FutureManager +from twinkle.server.state.models import FutureFailureRecord class _FakeBackend(StateBackend): @@ -74,7 +75,7 @@ async def update_atomic( return updated async def close(self) -> None: - self._store.clear() + pass async def health_check(self) -> bool: return True @@ -86,10 +87,15 @@ def manager(): async def _store(manager, request_id, status, *, replica_id=None, absolute_deadline=None): + failure = None + if status in ('failed', 'cancelled'): + failure = FutureFailureRecord( + reason_code='internal_error', message='boom', attribution='server') await manager.store_status( request_id, status, model_id='m1', + failure=failure, replica_id=replica_id, absolute_deadline=absolute_deadline, ) @@ -112,7 +118,9 @@ async def test_non_terminal_orphan_is_failed_not_deleted(manager): rec = await manager.get('r2') assert rec is not None # NOT deleted (Property 6) assert rec.status == 'failed' - assert rec.result['category'] == 'server' + assert rec.result is None + assert rec.failure.reason_code == 'orphaned_replica' + assert rec.failure.attribution == 'server' @pytest.mark.asyncio @@ -121,15 +129,17 @@ async def test_non_terminal_past_absolute_deadline_is_failed(manager): await manager.cleanup_expired(cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}) rec = await manager.get('r3') assert rec is not None and rec.status == 'failed' + assert rec.failure.reason_code == 'deadline_exceeded' @pytest.mark.asyncio -async def test_legacy_record_without_deadline_uses_expiration_timeout(manager): - await _store(manager, 'legacy', 'running', replica_id=None) +async def test_record_without_deadline_uses_expiration_timeout(manager): + await _store(manager, 'without-deadline', 'running', replica_id=None) with mock.patch('twinkle.server.state.future_manager.time.time', return_value=time.time() + 301): await manager.cleanup_expired(cutoff_time=time.time() + 10, alive_replica_ids=set()) - rec = await manager.get('legacy') + rec = await manager.get('without-deadline') assert rec is not None and rec.status == 'failed' + assert rec.failure.reason_code == 'deadline_exceeded' @pytest.mark.asyncio @@ -206,11 +216,14 @@ async def test_claim_seq_dedups_then_release_readmits(): async def test_cancel_drops_pending_but_never_running(): from twinkle.server.state.server_state import ServerState state = ServerState(backend=_FakeBackend()) - # pending -> cancel drops it to the terminal 'cancelled' state with a user payload. + # pending -> cancel drops it to a terminal domain failure. await state.store_future_status('rp', 'pending', 'm1') assert await state.cancel_future('rp') == {'cancelled': True, 'state': 'cancelled'} rec = await state.get_future('rp') - assert rec['status'] == 'cancelled' and rec['result']['error_code'] == 499 + assert rec['status'] == 'cancelled' + assert rec['result'] is None + assert rec['failure']['reason_code'] == 'cancelled' + assert rec['failure']['attribution'] == 'user' # running -> cancel is a no-op; in-flight work is never interrupted. await state.store_future_status('rr', 'running', 'm1') assert await state.cancel_future('rr') == {'cancelled': False, 'state': 'running'} @@ -220,8 +233,14 @@ async def test_cancel_drops_pending_but_never_running(): def test_cancelled_record_maps_to_error_envelope(): from twinkle.server.lifecycle.envelope import envelope_from_record - rec = {'status': 'cancelled', - 'result': {'error': 'Task cancelled by client', 'category': 'user', 'error_code': 499}} + rec = { + 'status': 'cancelled', + 'failure': { + 'reason_code': 'cancelled', + 'message': 'Task cancelled by client', + 'attribution': 'user', + }, + } env = envelope_from_record('rc', rec) assert env.status == 'cancelled' assert env.error is not None and env.error.error_code == 499 diff --git a/tests/server/state/test_managers.py b/tests/server/state/test_managers.py index 63fc24be..b607c9fc 100644 --- a/tests/server/state/test_managers.py +++ b/tests/server/state/test_managers.py @@ -7,6 +7,7 @@ from datetime import datetime, timezone from unittest import mock +from twinkle.server.exceptions import ResourceQuotaExceededError from twinkle.server.state import ServerState from twinkle.server.state.backend.memory_backend import RayActorBackend from twinkle.server.state.future_manager import FutureManager @@ -14,6 +15,7 @@ from twinkle.server.state.models import FutureRecord, ModelRecord, SamplingSessionRecord, SessionRecord from twinkle.server.state.sampling_manager import SamplingSessionManager from twinkle.server.state.session_manager import SessionManager +from .fake_backend import FakeBackend # ============================================================ # SessionManager Tests @@ -97,7 +99,7 @@ async def test_cleanup_expired(self, manager): await manager.add('new_sess', new_record) cutoff = now - 500 - removed_count = await manager.cleanup_expired(cutoff) + _, removed_count = await manager.collect_and_remove_expired(cutoff) assert removed_count == 1 assert await manager.get('old_sess') is None assert await manager.get('new_sess') is not None @@ -110,7 +112,7 @@ async def test_cleanup_expired_uses_created_at_fallback(self, manager): await manager.add('old_sess', record) cutoff = time.time() - 100 - removed_count = await manager.cleanup_expired(cutoff) + _, removed_count = await manager.collect_and_remove_expired(cutoff) assert removed_count == 1 @@ -148,11 +150,11 @@ async def test_remove(self, manager): @pytest.mark.asyncio async def test_token_limit_enforced(self, manager): - """Adding more models than per_token_model_limit should raise RuntimeError.""" + """Adding more models than the per-token quota raises a user quota error.""" for i in range(3): await manager.add(f'm{i}', ModelRecord(token='tok1')) - with pytest.raises(RuntimeError, match='Model limit exceeded'): + with pytest.raises(ResourceQuotaExceededError, match='Model quota exceeded'): await manager.add('m3', ModelRecord(token='tok1')) @pytest.mark.asyncio @@ -259,7 +261,7 @@ async def try_add(i: int) -> None: try: await state.register_model({'base_model': 'b'}, token='tok', model_id=f'm{i}') results.append(True) - except RuntimeError: + except ResourceQuotaExceededError: results.append(False) await asyncio.gather(*(try_add(i) for i in range(n))) @@ -276,7 +278,7 @@ async def test_remove_frees_token_slot(self): state = ServerState(backend=backend, per_token_model_limit=1) await state.register_model({'base_model': 'b'}, token='tok', model_id='m1') - with pytest.raises(RuntimeError): + with pytest.raises(ResourceQuotaExceededError): await state.register_model({'base_model': 'b'}, token='tok', model_id='m2') assert await state.unload_model('m1') is True @@ -295,10 +297,57 @@ async def test_rebuild_indexes_recovers_counter_from_records(self): await state._model_mgr.rebuild_indexes() await state.register_model({'base_model': 'b'}, token='tok', model_id='m3') - with pytest.raises(RuntimeError): + with pytest.raises(ResourceQuotaExceededError): await state.register_model({'base_model': 'b'}, token='tok', model_id='m4') +# ============================================================ +# Cluster-wide Processor Quota Tests +# ============================================================ + + +class TestProcessorQuota: + + @pytest.mark.asyncio + async def test_two_server_states_share_one_token_limit(self): + backend = FakeBackend() + states = [ServerState(backend=backend), ServerState(backend=backend)] + + async def reserve(i: int) -> bool: + try: + await states[i % 2].reserve_processor_quota( + 'token', f'p{i}', f's{i}', limit=3, lease_seconds=30.0) + return True + except ResourceQuotaExceededError: + return False + + accepted = await asyncio.gather(*(reserve(i) for i in range(12))) + assert sum(accepted) == 3 + + @pytest.mark.asyncio + async def test_reservation_is_idempotent_and_release_frees_slot(self): + state = ServerState(backend=FakeBackend()) + await state.reserve_processor_quota('token', 'p1', 's1', limit=1, lease_seconds=30.0) + await state.reserve_processor_quota('token', 'p1', 's1', limit=1, lease_seconds=30.0) + with pytest.raises(ResourceQuotaExceededError) as exc: + await state.reserve_processor_quota('token', 'p2', 's2', limit=1, lease_seconds=30.0) + assert exc.value.error_code == 429 + assert exc.value.category.value == 'user' + + await state.release_processor_quota('token', 'p1') + await state.release_processor_quota('token', 'p1') + await state.reserve_processor_quota('token', 'p2', 's2', limit=1, lease_seconds=30.0) + + @pytest.mark.asyncio + async def test_expired_worker_lease_is_reclaimed(self): + state = ServerState(backend=FakeBackend()) + with mock.patch('twinkle.server.state.server_state.time.time', return_value=100.0): + await state.reserve_processor_quota('token', 'dead', 's1', limit=1, lease_seconds=10.0) + with mock.patch('twinkle.server.state.server_state.time.time', return_value=111.0): + assert await state.renew_processor_quota('token', 'dead', lease_seconds=10.0) is False + await state.reserve_processor_quota('token', 'replacement', 's2', limit=1, lease_seconds=10.0) + + # ============================================================ # SamplingSessionManager Tests # ============================================================ @@ -334,7 +383,7 @@ async def test_cleanup_expired_by_age(self, manager): await manager.add('samp_new', record2) cutoff = time.time() - 100 - removed = await manager.cleanup_expired(cutoff) + removed = await manager.cleanup_expired(cutoff, []) assert removed == 1 assert await manager.get('samp_old') is None assert await manager.get('samp_new') is not None diff --git a/tests/server/state/test_update_atomic.py b/tests/server/state/test_update_atomic.py index 5c7ca7be..d03b3a79 100644 --- a/tests/server/state/test_update_atomic.py +++ b/tests/server/state/test_update_atomic.py @@ -1,7 +1,6 @@ -"""Cross-backend tests for ``StateBackend.update_atomic`` and ``set_nx(ttl)``. +"""Cross-backend tests for the Ray-actor and Redis state backends. -Exercises five contracts against each of the three production backends -(Memory, File, Redis): +Exercises atomic updates, leases, close semantics, and logical key prefixes: - read-transform-write returns the new value - ``transform`` returning ``None`` is a no-op and returns the existing value - ``ttl`` shapes the new value's expiry @@ -15,12 +14,15 @@ import asyncio import functools +import json import os import pytest import pytest_asyncio import uuid from typing import Any +from twinkle.server.deployment import twinkle_server_error_handler +from twinkle.server.state.backend.base import ConcurrencyError from twinkle.server.state.backend.memory_backend import RayActorBackend REDIS_URL = os.environ.get('TWINKLE_TEST_REDIS_URL', 'redis://localhost:6379/0') @@ -90,6 +92,43 @@ async def redis_backend(): await backend.close() +@pytest_asyncio.fixture(params=['memory', 'redis']) +async def backend_pair(request): + prefix = f'twinkle-contract-{uuid.uuid4().hex[:8]}::' + if request.param == 'redis': + if not _REDIS_AVAILABLE_AT_COLLECTION: + pytest.skip(f'Redis at {REDIS_URL} unreachable') + from twinkle.server.state.backend.redis_backend import RedisBackend + first = RedisBackend(REDIS_URL, key_prefix=prefix) + second = RedisBackend(REDIS_URL, key_prefix=prefix) + else: + first = RayActorBackend(key_prefix=prefix) + second = RayActorBackend(key_prefix=prefix) + yield first, second + for key in await second.keys('*'): + await second.delete(key) + await first.close() + await second.close() + + +# ---------- Shared backend contract -------------------------------------- # + + +@pytest.mark.asyncio +async def test_close_releases_handle_without_dropping_shared_state(backend_pair) -> None: + first, second = backend_pair + await first.set('shared', {'value': 1}) + await first.close() + assert await second.get('shared') == {'value': 1} + + +@pytest.mark.asyncio +async def test_key_prefix_is_hidden_from_logical_keys(backend_pair) -> None: + first, _ = backend_pair + await first.set('session::one', 1) + assert await first.keys('session::*') == ['session::one'] + + # ---------- Memory backend ----------------------------------------------- # @@ -183,3 +222,39 @@ async def test_redis_update_atomic_respects_ttl(redis_backend) -> None: assert await redis_backend.get('leased') == 'holder' await asyncio.sleep(1.5) assert await redis_backend.get('leased') is None + + +@_redis_skip +@pytest.mark.asyncio +async def test_redis_retry_exhaustion_maps_to_http_503(redis_backend) -> None: + import redis + from starlette.requests import Request + + sync_client = redis.Redis.from_url(REDIS_URL, decode_responses=True) + real_key = redis_backend._make_key('always-contended') + calls = 0 + + def force_watch_conflict(current: Any | None) -> int: + nonlocal calls + calls += 1 + sync_client.set(real_key, json.dumps(calls)) + return int(current or 0) + 1 + + try: + with pytest.raises(ConcurrencyError) as caught: + await redis_backend.update_atomic('always-contended', force_watch_conflict) + finally: + sync_client.close() + + assert calls == 16 + request = Request({'type': 'http', 'method': 'POST', 'path': '/', 'headers': []}) + request.state.request_id = 'req-contention' + response = await twinkle_server_error_handler(request, caught.value) + payload = json.loads(response.body) + assert response.status_code == 503 + assert payload == { + 'error': "update_atomic exhausted 16 retries on key 'always-contended'", + 'category': 'server', + 'error_code': 503, + 'request_id': 'req-contention', + } diff --git a/tests/server/static/test_no_twinkle_http_exception.py b/tests/server/static/test_no_twinkle_http_exception.py new file mode 100644 index 00000000..28a67a39 --- /dev/null +++ b/tests/server/static/test_no_twinkle_http_exception.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import ast +from pathlib import Path + +_SERVER = Path(__file__).resolve().parents[3] / 'src' / 'twinkle' / 'server' +_TWINKLE_HANDLERS = ( + _SERVER / 'gateway' / 'twinkle_handlers.py', + _SERVER / 'model' / 'twinkle_handlers.py', + _SERVER / 'sampler' / 'twinkle_handlers.py', + _SERVER / 'processor' / 'twinkle_handlers.py', +) + + +def _status_code(call: ast.Call) -> int | None: + if call.args and isinstance(call.args[0], ast.Constant) and isinstance(call.args[0].value, int): + return call.args[0].value + for keyword in call.keywords: + if keyword.arg == 'status_code' and isinstance(keyword.value, ast.Constant): + return keyword.value.value if isinstance(keyword.value.value, int) else None + return None + + +def test_twinkle_handlers_have_no_http_exception_bypass_except_iterator_410() -> None: + violations: list[str] = [] + allowed_410 = 0 + for path in _TWINKLE_HANDLERS: + tree = ast.parse(path.read_text(encoding='utf-8'), filename=str(path)) + for node in ast.walk(tree): + if not isinstance(node, ast.Raise) or not isinstance(node.exc, ast.Call): + continue + func = node.exc.func + if not isinstance(func, ast.Name) or func.id != 'HTTPException': + continue + status_code = _status_code(node.exc) + if path.parent.name == 'processor' and status_code == 410: + allowed_410 += 1 + else: + violations.append(f'{path.relative_to(_SERVER)}:{node.lineno} status={status_code}') + + assert allowed_410 == 1 + assert violations == [] diff --git a/tests/server/utils/test_task_queue_mixin.py b/tests/server/utils/test_task_queue_mixin.py index 98aaf5e7..fd893cd7 100644 --- a/tests/server/utils/test_task_queue_mixin.py +++ b/tests/server/utils/test_task_queue_mixin.py @@ -19,6 +19,7 @@ async def store_future_status(self, *args, **kwargs): self._latest[request_id] = { 'status': status, 'result': kwargs.get('result'), + 'failure': (kwargs['failure'].model_dump() if kwargs.get('failure') is not None else None), 'queue_state': kwargs.get('queue_state'), 'queue_state_reason': kwargs.get('queue_state_reason'), } @@ -203,14 +204,15 @@ async def work(): finally: await queue._compute_worker.stop() - assert failed[-1]['result']['category'] == 'user' - assert 'traceback' not in failed[-1]['result'] + failure = failed[-1]['failure'] + assert failure.reason_code == 'request_rejected' + assert failure.attribution == 'user' + assert failure.diagnostic is None @pytest.mark.asyncio -async def test_typed_server_error_keeps_its_status_and_category(): - """A TwinkleServerError (e.g. ResourceNotFoundError) must keep its own 404/user - classification instead of collapsing to a generic 500/server.""" +async def test_typed_server_error_keeps_its_domain_reason_and_attribution(): + """A typed user failure must retain its domain meaning in persisted state.""" from twinkle.server.exceptions import ResourceNotFoundError queue = _DummyQueue() @@ -229,10 +231,10 @@ async def work(): finally: await queue._compute_worker.stop() - assert failed[-1]['result']['error_code'] == 404 - assert failed[-1]['result']['category'] == 'user' - # A user rejection carries no traceback. - assert 'traceback' not in failed[-1]['result'] + failure = failed[-1]['failure'] + assert failure.reason_code == 'resource_not_found' + assert failure.attribution == 'user' + assert failure.diagnostic is None @pytest.mark.asyncio diff --git a/tests/server/validation/test_preflight.py b/tests/server/validation/test_preflight.py index 81e5832b..330a8f4d 100644 --- a/tests/server/validation/test_preflight.py +++ b/tests/server/validation/test_preflight.py @@ -12,9 +12,9 @@ import pytest from types import SimpleNamespace -from twinkle.server.exceptions import RequestRejectedError +from twinkle.server.exceptions import EndpointUnavailableError, RequestRejectedError from twinkle.server.lifecycle.submit import backend_kwargs, run_submit -from twinkle.server.validation import BackendCapability, EndpointUnavailableError, assert_request_supported +from twinkle.server.validation import BackendCapability, assert_request_supported from twinkle.server.validation.backend_compat import resolve_backend from twinkle_client.types import model as model_types diff --git a/tests/twinkle_client/test_error_parsing.py b/tests/twinkle_client/test_error_parsing.py index 9c322290..76b688a1 100644 --- a/tests/twinkle_client/test_error_parsing.py +++ b/tests/twinkle_client/test_error_parsing.py @@ -35,16 +35,18 @@ def test_structured_error_reads_top_level_fields(): assert exc.value.error_code == 422 assert exc.value.category == 'user' assert exc.value.request_id == 'req-7' + assert exc.value.details is None + assert exc.value.traceback is None assert 'bad input' in str(exc.value) def test_detail_only_error_falls_back_to_unknown_category(): - """R3#7: FastAPI's built-in {detail: ...} maps to category='Unknown'.""" + """A non-ErrorPayload JSON body uses the lowercase unknown category.""" resp = _Resp(404, body={'detail': 'Not Found'}) with pytest.raises(TwinkleHTTPError) as exc: _handle_response(resp) assert exc.value.status_code == 404 - assert exc.value.category == 'Unknown' + assert exc.value.category == 'unknown' assert exc.value.error_code is None assert 'Not Found' in str(exc.value) @@ -53,10 +55,46 @@ def test_non_json_body_falls_back_to_text(): resp = _Resp(500, body=None, text='raw traceback text') with pytest.raises(TwinkleHTTPError) as exc: _handle_response(resp) - assert exc.value.category == 'Unknown' + assert exc.value.category == 'unknown' assert 'raw traceback text' in str(exc.value) +def test_validation_details_are_preserved(): + details = [{'loc': ['body', 'items', 0], 'msg': 'invalid', 'type': 'value_error'}] + resp = _Resp( + 422, + body={ + 'error': 'request validation failed', + 'category': 'user', + 'error_code': 422, + 'request_id': 'req-details', + 'details': details, + }, + ) + with pytest.raises(TwinkleHTTPError) as exc: + _handle_response(resp) + assert exc.value.details == details + assert exc.value.traceback is None + + +def test_server_traceback_is_preserved(): + traceback_text = 'Traceback (most recent call last):\n File "/srv/app.py", line 1\nRuntimeError: boom' + resp = _Resp( + 500, + body={ + 'error': 'RuntimeError: boom', + 'category': 'server', + 'error_code': 500, + 'request_id': 'req-trace', + 'traceback': traceback_text, + }, + ) + with pytest.raises(TwinkleHTTPError) as exc: + _handle_response(resp) + assert exc.value.traceback == traceback_text + assert exc.value.details is None + + def test_410_raises_stop_iteration_not_http_error(): """R3#9: 410 keeps raising StopIteration, not an HTTP error.""" resp = _Resp(410, body={'detail': 'exhausted'}) From 7062b83eafe2e4c9ebca3f56d027010326c75cd9 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 21 Sep 2026 01:10:59 +0000 Subject: [PATCH 25/32] =?UTF-8?q?refactor(server):=20module=20boundaries?= =?UTF-8?q?=20=E2=80=94=20server/utils=20decomposition,=20ServerState/Back?= =?UTF-8?q?endGate=20split=20(server-module-boundaries=20Track=20C)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Structural (behaviour-preserving) module-boundary work: - server/utils/ dissolved: task_queue/, session_resource/ promoted to server/; auth.py -> middleware/auth.py; task_errors.py -> server/; backend_dispatch.py -> config/; ray_serve_patch.py -> twinkle/patch/ray_serve.py. device/template utils stay. - common/ removed: datum.py -> model/tinker_datum.py, router.py -> model/routing.py. - validation/errors.py merged into deployment.validation_error_handler; validation/ now holds only backend_compat. - ServerState split into ResourceCleanupCoordinator + ResourceCountPublisher (idempotency guard moved with the impl); actor_name -> cache_key. - BackendGate extracted from TaskQueueMixin (pure callable wrapper); worker owns notify_new_task + the single queue-depth writer. - tinker_load path resolution moved to the handler; backends take checkpoint_name/ output_dir and no longer import server.checkpoint. - gateway services.py -> use_cases.py (dropped one-line forwards); supported_model_names public property; per-instance template cache; gateway/routes.py route constants. - init_twinkle_runtime -> server/runtime.py (with ncpu_proc_per_node passthrough). - tq_utils.py -> data_format/tq_fields.py (+ TQ field constants; async_rl shim removed). - lifecycle/protocols.py declares the host contract; run_submit/input_metrics annotated. - utils bucket export shrunk; telemetry middleware.py split into metrics.py + http_middleware.py; dropped stale re-exports (FullModeBusyError, PersistenceConfig, TelemetryConfig, _resolve_client_save_dir). fix(telemetry): per-deployment metric caches moved onto MetricsRegistry instance so MetricsRegistry.reset() invalidates them (adapters no longer keep NoOp instruments). Also: added tests (tq_fields, proxy, runtime, protocols, metrics-cache-invalidation), updated imports/test harnesses, and removed spec-clause reference annotations from code comments/docstrings. Verified: regression baseline 291 passed / 16 skipped; full 2x2x3 E2E (transformers & megatron x twinkle & tinker x sft/dpo/grpo) all passed; flake8/yapf clean. --- src/twinkle/data_format/__init__.py | 2 + .../{tq_utils.py => data_format/tq_fields.py} | 37 +- .../ray_serve_patch.py => patch/ray_serve.py} | 2 +- src/twinkle/server/checkpoint/__init__.py | 3 +- src/twinkle/server/common/__init__.py | 10 - src/twinkle/server/config/application_spec.py | 2 +- .../{utils => config}/backend_dispatch.py | 0 src/twinkle/server/config/server_config.py | 2 +- src/twinkle/server/data_plane/store.py | 2 +- src/twinkle/server/deployment.py | 108 ++++-- src/twinkle/server/gateway/app.py | 20 +- src/twinkle/server/gateway/openai_handlers.py | 25 +- src/twinkle/server/gateway/proxy.py | 8 +- src/twinkle/server/gateway/routes.py | 26 ++ src/twinkle/server/gateway/tinker_handlers.py | 19 +- .../server/gateway/twinkle_handlers.py | 17 +- .../gateway/{services.py => use_cases.py} | 19 +- .../server/launcher/server_launcher.py | 2 +- src/twinkle/server/lifecycle/envelope.py | 2 +- src/twinkle/server/lifecycle/protocols.py | 60 ++++ src/twinkle/server/lifecycle/submit.py | 15 +- src/twinkle/server/middleware/__init__.py | 6 + .../server/{utils => middleware}/auth.py | 0 src/twinkle/server/model/app.py | 29 +- .../server/model/backends/megatron_model.py | 24 +- .../server/model/backends/mock_model.py | 2 +- .../model/backends/transformers_model.py | 26 +- .../{common/router.py => model/routing.py} | 1 - .../datum.py => model/tinker_datum.py} | 0 src/twinkle/server/model/tinker_handlers.py | 25 +- src/twinkle/server/model/twinkle_handlers.py | 21 +- src/twinkle/server/processor/app.py | 21 +- .../server/processor/twinkle_handlers.py | 20 +- src/twinkle/server/runtime.py | 52 +++ src/twinkle/server/sampler/app.py | 12 +- src/twinkle/server/sampler/tinker_handlers.py | 10 +- .../server/sampler/twinkle_handlers.py | 28 +- .../{utils => }/session_resource/__init__.py | 0 .../{utils => }/session_resource/adapter.py | 0 .../{utils => }/session_resource/base.py | 0 .../{utils => }/session_resource/processor.py | 0 src/twinkle/server/state/__init__.py | 7 +- .../server/state/cleanup_coordinator.py | 247 +++++++++++++ src/twinkle/server/state/count_publisher.py | 78 +++++ src/twinkle/server/state/server_state.py | 327 +++++------------- src/twinkle/server/{utils => }/task_errors.py | 0 .../server/{utils => }/task_queue/__init__.py | 2 +- src/twinkle/server/task_queue/backend_gate.py | 121 +++++++ .../server/{utils => }/task_queue/config.py | 0 .../server/{utils => }/task_queue/mixin.py | 111 ++---- .../{utils => }/task_queue/rate_limiter.py | 2 +- .../server/{utils => }/task_queue/types.py | 0 .../server/{utils => }/task_queue/worker.py | 27 +- src/twinkle/server/telemetry/__init__.py | 2 - .../server/telemetry/http_middleware.py | 56 +++ src/twinkle/server/telemetry/metrics.py | 150 +++++++- src/twinkle/server/telemetry/middleware.py | 208 ----------- src/twinkle/server/utils/__init__.py | 11 +- src/twinkle/server/validation/__init__.py | 5 - .../server/validation/backend_compat.py | 9 + src/twinkle/server/validation/errors.py | 77 ----- src/twinkle_agentic/async_rl/data_plane.py | 3 +- src/twinkle_agentic/async_rl/pipeline.py | 2 +- src/twinkle_agentic/async_rl/tq_utils.py | 29 -- src/twinkle_client/types/training.py | 2 +- tests/data_format/test_tq_fields.py | 58 ++++ tests/infra/test_ray_get_timeout.py | 6 +- tests/server/conftest.py | 2 +- tests/server/contract/client_api_harness.py | 16 +- .../contract/test_client_api_contract.py | 4 +- tests/server/contract/test_error_wire.py | 4 +- tests/server/gateway/test_openai_handlers.py | 15 +- tests/server/gateway/test_proxy.py | 66 ++++ .../server/integration/test_actor_recovery.py | 6 +- .../integration/test_blocking_boundary.py | 31 +- tests/server/integration/test_dpo_e2e.py | 2 +- .../server/integration/test_full_param_e2e.py | 2 +- .../integration/test_nccl_safe_tinker_e2e.py | 12 +- .../integration/test_nccl_safe_twinkle_e2e.py | 7 +- tests/server/integration/test_sft_e2e.py | 2 +- tests/server/lifecycle/test_envelope.py | 4 +- .../lifecycle/test_envelope_coverage.py | 6 +- .../lifecycle/test_preflight_rejection.py | 14 +- tests/server/lifecycle/test_protocols.py | 26 ++ .../lifecycle/test_retrieve_endpoint.py | 10 +- .../server/lifecycle/test_run_submit_dedup.py | 2 +- tests/server/lifecycle/test_static_guards.py | 12 +- .../server/lifecycle/test_submit_peek_e2e.py | 13 +- tests/server/lifecycle/test_timing_bounds.py | 16 +- .../test_tinker_retrieve_regression.py | 6 +- .../server/model/test_tinker_compat_output.py | 2 +- tests/server/model/test_tinker_handlers.py | 8 +- .../server/model/test_twinkle_async_inputs.py | 6 +- .../server/sampler/test_twinkle_async_rows.py | 2 +- .../server/session_resource/test_contract.py | 18 +- tests/server/state/test_error_payload.py | 4 +- tests/server/state/test_future_lifecycle.py | 6 +- tests/server/state/test_leader_election.py | 2 +- .../server/static/backend_call_exemptions.py | 6 +- tests/server/static/test_no_degraded_path.py | 2 +- .../static/test_no_direct_backend_call.py | 2 - .../static/test_utils_bucket_is_light.py | 34 ++ .../test_metrics_cache_invalidation.py | 20 ++ tests/server/test_gateway_services.py | 6 +- tests/server/test_runtime.py | 42 +++ tests/server/utils/task_queue/test_config.py | 2 +- tests/server/utils/test_task_errors.py | 2 +- tests/server/utils/test_task_queue_mixin.py | 12 +- tests/server/validation/test_request_wire.py | 6 +- tests/twinkle_client/test_error_parsing.py | 8 +- tests/twinkle_client/test_future_layer.py | 12 +- tests/twinkle_client/test_types_contract.py | 6 +- 112 files changed, 1640 insertions(+), 1044 deletions(-) rename src/twinkle/{tq_utils.py => data_format/tq_fields.py} (51%) rename src/twinkle/{server/utils/ray_serve_patch.py => patch/ray_serve.py} (97%) delete mode 100644 src/twinkle/server/common/__init__.py rename src/twinkle/server/{utils => config}/backend_dispatch.py (100%) create mode 100644 src/twinkle/server/gateway/routes.py rename src/twinkle/server/gateway/{services.py => use_cases.py} (78%) create mode 100644 src/twinkle/server/lifecycle/protocols.py create mode 100644 src/twinkle/server/middleware/__init__.py rename src/twinkle/server/{utils => middleware}/auth.py (100%) rename src/twinkle/server/{common/router.py => model/routing.py} (98%) rename src/twinkle/server/{common/datum.py => model/tinker_datum.py} (100%) create mode 100644 src/twinkle/server/runtime.py rename src/twinkle/server/{utils => }/session_resource/__init__.py (100%) rename src/twinkle/server/{utils => }/session_resource/adapter.py (100%) rename src/twinkle/server/{utils => }/session_resource/base.py (100%) rename src/twinkle/server/{utils => }/session_resource/processor.py (100%) create mode 100644 src/twinkle/server/state/cleanup_coordinator.py create mode 100644 src/twinkle/server/state/count_publisher.py rename src/twinkle/server/{utils => }/task_errors.py (100%) rename src/twinkle/server/{utils => }/task_queue/__init__.py (91%) create mode 100644 src/twinkle/server/task_queue/backend_gate.py rename src/twinkle/server/{utils => }/task_queue/config.py (100%) rename src/twinkle/server/{utils => }/task_queue/mixin.py (80%) rename src/twinkle/server/{utils => }/task_queue/rate_limiter.py (99%) rename src/twinkle/server/{utils => }/task_queue/types.py (100%) rename src/twinkle/server/{utils => }/task_queue/worker.py (95%) create mode 100644 src/twinkle/server/telemetry/http_middleware.py delete mode 100644 src/twinkle/server/telemetry/middleware.py delete mode 100644 src/twinkle/server/validation/errors.py delete mode 100644 src/twinkle_agentic/async_rl/tq_utils.py create mode 100644 tests/data_format/test_tq_fields.py create mode 100644 tests/server/gateway/test_proxy.py create mode 100644 tests/server/lifecycle/test_protocols.py create mode 100644 tests/server/static/test_utils_bucket_is_light.py create mode 100644 tests/server/telemetry/test_metrics_cache_invalidation.py create mode 100644 tests/server/test_runtime.py diff --git a/src/twinkle/data_format/__init__.py b/src/twinkle/data_format/__init__.py index 93946382..01a9b629 100644 --- a/src/twinkle/data_format/__init__.py +++ b/src/twinkle/data_format/__init__.py @@ -4,4 +4,6 @@ from .message import Message, Tool, ToolCall from .output import LossOutput, ModelOutput from .sampling import SampledSequence, SampleResponse, SamplingMask, SamplingParams +from .tq_fields import (REQUIRED_MODEL_INPUT_FIELDS, ROLLOUT_TRAIN_FIELDS, TRANSFORMERS_INPUT_FIELDS, + columns_to_tq_fields, rows_to_tq_fields) from .trajectory import Trajectory, attach_user_data, pack_user_data, pack_value, user_data_get diff --git a/src/twinkle/tq_utils.py b/src/twinkle/data_format/tq_fields.py similarity index 51% rename from src/twinkle/tq_utils.py rename to src/twinkle/data_format/tq_fields.py index f7c34f07..a1eff271 100644 --- a/src/twinkle/tq_utils.py +++ b/src/twinkle/data_format/tq_fields.py @@ -1,10 +1,45 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Small TransferQueue packing helpers shared by both async-RL modes.""" +"""TransferQueue field packing and the field-name schema both async-RL modes share. + +Lives in ``data_format/`` because "rows/columns -> TensorDict" is a data-format +conversion, alongside ``input_feature`` / ``trajectory`` / ``encoding`` / ``message`` / +``output`` / ``sampling``. It used to sit at the package root as ``twinkle/tq_utils.py`` +-- the only domain module there, unreachable via ``twinkle.``, under an unexplained +abbreviation and a ``_utils`` suffix that undersold what it does (it validates field +consistency and raises). + +The field lists moved here from ``twinkle_agentic/async_rl/tq_utils.py``: the schema and +the packing logic belong in one file, and that shim existed only to re-export this +module. This deliberately means ``twinkle`` holds the RL training field names +(``logprobs`` / ``rewards`` / ``advantages`` / ``returns``) while ``twinkle_agentic`` +only consumes them. + +``torch`` / ``tensordict`` stay inside the functions: ``tensordict`` arrives with +``TransferQueue``, which is only in the ``async-rl`` extra, so this module must import +cleanly without it. Do NOT hoist them. +""" from __future__ import annotations from numbers import Number from typing import Any +TRANSFORMERS_INPUT_FIELDS = ( + 'input_ids', + 'labels', + 'attention_mask', + 'position_ids', + 'cu_seqlens', + 'completion_mask', + 'pixel_values', + 'image_grid_thw', + 'video_pixel_values', + 'video_grid_thw', + 'input_features', + 'feature_attention_mask', +) +REQUIRED_MODEL_INPUT_FIELDS = ('input_ids', 'labels', 'attention_mask', 'position_ids') +ROLLOUT_TRAIN_FIELDS = (*TRANSFORMERS_INPUT_FIELDS, 'logprobs', 'rewards', 'advantages', 'returns') + def rows_to_tq_fields(rows: list[dict[str, Any]]): from tensordict import TensorDict diff --git a/src/twinkle/server/utils/ray_serve_patch.py b/src/twinkle/patch/ray_serve.py similarity index 97% rename from src/twinkle/server/utils/ray_serve_patch.py rename to src/twinkle/patch/ray_serve.py index 69c385dd..dabae754 100644 --- a/src/twinkle/server/utils/ray_serve_patch.py +++ b/src/twinkle/patch/ray_serve.py @@ -138,4 +138,4 @@ def get_runtime_env_for_patches() -> dict: Returns: dict: Ray runtime_env configuration """ - return {'worker_process_setup_hook': ('twinkle.server.utils.ray_serve_patch._apply_patch_in_worker_process')} + return {'worker_process_setup_hook': ('twinkle.patch.ray_serve._apply_patch_in_worker_process')} diff --git a/src/twinkle/server/checkpoint/__init__.py b/src/twinkle/server/checkpoint/__init__.py index e8bfccd1..e6a129aa 100644 --- a/src/twinkle/server/checkpoint/__init__.py +++ b/src/twinkle/server/checkpoint/__init__.py @@ -10,7 +10,7 @@ from twinkle.server.checkpoint import ( create_checkpoint_manager, create_training_run_manager, BaseCheckpointManager, BaseTrainingRunManager, BaseFileManager, - validate_user_path, validate_ownership, _resolve_client_save_dir, + validate_user_path, validate_ownership, TRAIN_RUN_INFO_FILENAME, TWINKLE_DEFAULT_SAVE_DIR, ) """ @@ -30,5 +30,4 @@ 'BaseTrainingRunManager', 'validate_user_path', 'validate_ownership', - '_resolve_client_save_dir', ] diff --git a/src/twinkle/server/common/__init__.py b/src/twinkle/server/common/__init__.py deleted file mode 100644 index 5fa5109d..00000000 --- a/src/twinkle/server/common/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) ModelScope Contributors. All rights reserved. -from .datum import datum_to_input_feature, extract_rl_features_for_loss, input_feature_to_datum -from .router import StickyLoraRequestRouter - -__all__ = [ - 'datum_to_input_feature', - 'extract_rl_features_for_loss', - 'input_feature_to_datum', - 'StickyLoraRequestRouter', -] diff --git a/src/twinkle/server/config/application_spec.py b/src/twinkle/server/config/application_spec.py index 3117d65c..397fe8b0 100644 --- a/src/twinkle/server/config/application_spec.py +++ b/src/twinkle/server/config/application_spec.py @@ -15,7 +15,7 @@ from pydantic import BaseModel, ConfigDict, Field, model_validator from typing import Any, Literal -from twinkle.server.utils.task_queue.config import TaskQueueConfig +from twinkle.server.task_queue.config import TaskQueueConfig # Env var keys the launcher sets from the gateway ``server_config`` so that any # Ray worker (model / sampler / processor), not just the gateway, applies the diff --git a/src/twinkle/server/utils/backend_dispatch.py b/src/twinkle/server/config/backend_dispatch.py similarity index 100% rename from src/twinkle/server/utils/backend_dispatch.py rename to src/twinkle/server/config/backend_dispatch.py diff --git a/src/twinkle/server/config/server_config.py b/src/twinkle/server/config/server_config.py index 15206c7f..fd5da6c4 100644 --- a/src/twinkle/server/config/server_config.py +++ b/src/twinkle/server/config/server_config.py @@ -19,7 +19,7 @@ from typing import Any from twinkle.server.exceptions import ConfigParseError -from twinkle.server.utils.task_queue.config import TaskQueueConfig +from twinkle.server.task_queue.config import TaskQueueConfig from .application_spec import ApplicationSpec, HttpOptions from .persistence import PersistenceConfig from .telemetry import TelemetryConfig diff --git a/src/twinkle/server/data_plane/store.py b/src/twinkle/server/data_plane/store.py index 81fe7fa6..ad8a66fe 100644 --- a/src/twinkle/server/data_plane/store.py +++ b/src/twinkle/server/data_plane/store.py @@ -5,7 +5,7 @@ import uuid from typing import Any -from twinkle.tq_utils import rows_to_tq_fields +from twinkle.data_format import rows_to_tq_fields from twinkle_client.common.json_utils import json_safe from twinkle_client.types.component import DataRef diff --git a/src/twinkle/server/deployment.py b/src/twinkle/server/deployment.py index 239686bf..792ae6d3 100644 --- a/src/twinkle/server/deployment.py +++ b/src/twinkle/server/deployment.py @@ -31,15 +31,15 @@ from collections.abc import Awaitable, Callable from contextlib import asynccontextmanager from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse from ray import serve from typing import Any from twinkle.server.exceptions import TwinkleServerError -from twinkle.server.telemetry.middleware import create_metrics_middleware +from twinkle.server.middleware.auth import verify_request_token +from twinkle.server.telemetry.http_middleware import create_metrics_middleware from twinkle.server.telemetry.tracing import create_tracing_middleware -from twinkle.server.utils.auth import verify_request_token -from twinkle.server.validation.errors import register_validation_error_handler from twinkle.utils.logger import get_logger from twinkle_client.types.errors import ErrorCategory, ErrorPayload @@ -68,6 +68,68 @@ async def twinkle_server_error_handler(request: Request, exc: TwinkleServerError return JSONResponse(status_code=exc.error_code, content=payload.model_dump(mode='json', exclude_none=True)) +# A body can produce hundreds of errors (one per element of a mis-typed tensor), and a +# response listing all of them helps nobody while costing bandwidth on every retry. +_MAX_VALIDATION_DETAILS = 20 + + +def _validation_detail(error: dict[str, Any]) -> dict[str, Any]: + """One pydantic error as a JSON-safe detail entry.""" + location = [str(part) for part in error.get('loc', ())] + return { + 'field': location[-1] if location else '', + 'path': '.'.join(location), + 'type': error.get('type', ''), + 'message': error.get('msg', ''), + } + + +def _validation_summary(errors: list[dict[str, Any]]) -> str: + fields = [] + for error in errors: + path = '.'.join(str(part) for part in error.get('loc', ())) + if path and path not in fields: + fields.append(path) + shown = ', '.join(fields[:_MAX_VALIDATION_DETAILS]) or 'request body' + suffix = '' if len(fields) <= _MAX_VALIDATION_DETAILS else f' (+{len(fields) - _MAX_VALIDATION_DETAILS} more)' + return f'Request body validation failed for: {shown}{suffix}' + + +def _validation_mentions_unknown_field(errors: list[dict[str, Any]]) -> bool: + return any(error.get('type') == 'extra_forbidden' for error in errors) + + +async def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse: + """Map a body validation failure to a 422 carrying an ``ErrorPayload``. + + Lives next to ``twinkle_server_error_handler`` because both are the same concern -- + the wire shape of a failure -- and both are registered by ``build_deployment_app`` + for all four deployments. It used to sit in ``validation/``, whose ``__init__`` + docstring admitted the fit was awkward ("another half of the story"). + + FastAPI's default handler answers with ``{"detail": [...]}``, a second error shape + on the wire; this makes the Model, Sampler and Processor deployments answer + identically to every other twinkle failure. The per-field ``details`` name the + offending field, its path, and why it was rejected; there is no traceback because a + rejected body is the caller's problem, not a crash. + """ + errors = list(exc.errors()) + message = _validation_summary(errors) + if _validation_mentions_unknown_field(errors): + # An unknown top-level field is what an older client looks like against a newer + # server, so say so instead of leaving the caller to infer it from a field list. + message += ('. Unknown fields are rejected; if this worked before, upgrade ' + 'twinkle-kit on the client to match the server version.') + payload = ErrorPayload( + error=message[:1024], + category=ErrorCategory.User, + error_code=422, + request_id=getattr(request.state, 'request_id', None) or '', + details=[_validation_detail(error) for error in errors[:_MAX_VALIDATION_DETAILS]], + ) + return JSONResponse(status_code=422, content=payload.model_dump(mode='json', exclude_none=True)) + + def get_servable() -> Any: """The single definition of the servable-object accessor used by every builder. @@ -151,7 +213,7 @@ async def lifespan(app: FastAPI): # Request-body validation failures answer with the same ``ErrorPayload`` shape as # every other error, registered here so all deployments behave identically rather # than each app keeping (or forgetting) its own copy. - register_validation_error_handler(app) + app.add_exception_handler(RequestValidationError, validation_error_handler) # Registration order matters: FastAPI runs middleware LIFO, so the LAST # registered wraps the outermost layer. Register cleanup (if any) first so @@ -240,31 +302,6 @@ def bind_deployment( return deployment_cls.options(**deploy_options).bind(*bind_args, **(bind_kwargs or {})) -def init_twinkle_runtime( - is_mock: bool, - nproc_per_node: int, - device_group: Any, - device_mesh_dict: dict[str, Any], -) -> Any | None: - """Initialize the Twinkle distributed runtime and build a DeviceMesh. - - Shared by ModelManagement and SamplerManagement ``__init__``. - Returns ``None`` for mock backends (CPU-only, no device mesh). - """ - import twinkle - from twinkle import DeviceMesh - - if is_mock: - twinkle.initialize( - mode='ray', nproc_per_node=nproc_per_node, ncpu_proc_per_node=1, groups=[device_group], lazy_collect=False) - return None - - twinkle.initialize(mode='ray', nproc_per_node=nproc_per_node, groups=[device_group], lazy_collect=False) - if 'mesh_dim_names' in device_mesh_dict: - return DeviceMesh(**device_mesh_dict) - return DeviceMesh.from_sizes(**device_mesh_dict) - - class LazyCleanupMixin: """Single source of the lazy first-request ServerState cleanup-start behavior. @@ -272,13 +309,22 @@ class LazyCleanupMixin: ``_ensure_state_cleanup_started`` methods collapse into one. The method name is preserved so existing call sites (``_on_request_start``, ``_ensure_sticky``, the Gateway cleanup middleware) are unchanged. + + ``_state_cleanup_started`` is declared here with a class-level default so the four + deployment classes need no ``__init__`` change and the ``getattr`` fallback can go: + only ``GatewayServer`` used to initialise it explicitly, and Model/Sampler/Processor + relied on ``getattr``'s default. The first write of ``True`` shadows the + class attribute on the instance -- expected, since the class attribute is only a + default. """ + _state_cleanup_started: bool = False + async def _ensure_state_cleanup_started(self) -> None: - if getattr(self, '_state_cleanup_started', False): + if self._state_cleanup_started: return try: - # Idempotent via ServerState's own ``_cleanup_running`` guard. + # Idempotent via the ResourceCleanupCoordinator's own start guard. await self.state.start_cleanup_task() except Exception as e: logger.warning(f'Failed to start ServerState cleanup task: {e}') diff --git a/src/twinkle/server/gateway/app.py b/src/twinkle/server/gateway/app.py index bdf02065..74070e40 100644 --- a/src/twinkle/server/gateway/app.py +++ b/src/twinkle/server/gateway/app.py @@ -17,8 +17,8 @@ from twinkle.utils.logger import get_logger from .openai_handlers import _register_openai_routes from .proxy import ServiceProxy -from .tinker_handlers import _register_tinker_routes -from .twinkle_handlers import _register_twinkle_routes +from .tinker_handlers import _register_gateway_tinker_routes +from .twinkle_handlers import _register_gateway_twinkle_routes logger = get_logger() @@ -44,6 +44,18 @@ def __init__(self, self._supported_model_names = frozenset(m.model_name for m in self.supported_models) self._modelscope_config_lock = asyncio.Lock() self._state_cleanup_started = False + # Per-instance, not module-level: a process-global set never got cleared + # across replica rebuilds and made the OpenAI template tests non-isolatable. + self._template_initialized: set[str] = set() + + @property + def supported_model_names(self) -> frozenset[str]: + """Base-model names this gateway accepts. + + Public because ``openai_handlers._resolve_base_model`` reads it; it used to + reach into ``gateway._supported_model_names`` directly. + """ + return self._supported_model_names @staticmethod def _normalize_server_state_args(server_config: Any) -> dict[str, Any]: @@ -116,8 +128,8 @@ def build_gateway_app(deploy_options: dict[str, Any], # because it has no per-handler request hook, so the lazy-cleanup middleware # must cover every route (and stays innermost). def register_routes(app: FastAPI, get_self: Any) -> None: - _register_tinker_routes(app, get_self) - _register_twinkle_routes(app, get_self) + _register_gateway_tinker_routes(app, get_self) + _register_gateway_twinkle_routes(app, get_self) _register_openai_routes(app, get_self) async def _on_shutdown(servable: Any) -> None: diff --git a/src/twinkle/server/gateway/openai_handlers.py b/src/twinkle/server/gateway/openai_handlers.py index cb33f161..11030e06 100644 --- a/src/twinkle/server/gateway/openai_handlers.py +++ b/src/twinkle/server/gateway/openai_handlers.py @@ -24,6 +24,7 @@ from twinkle.server.utils import get_template_for_model from twinkle.utils.logger import get_logger +from . import routes from .openai_bridge import make_error, translate_chat_request, translate_response, translate_stream_chunk logger = get_logger() @@ -87,7 +88,7 @@ async def chat_completions( # Non-streaming: proxy to /twinkle/sample, translate response response = await self.proxy.proxy_request( request, - endpoint='twinkle/sample', + endpoint=routes.TWINKLE_SAMPLE, base_model=base_model, service_type='sampler', body_override=body_bytes, @@ -121,7 +122,7 @@ async def _sse_generator(): try: async for line in self.proxy.proxy_request_stream( request, - endpoint='twinkle/sample_stream', + endpoint=routes.TWINKLE_SAMPLE_STREAM, base_model=base_model, service_type='sampler', body_override=body_bytes, @@ -195,12 +196,12 @@ async def _resolve_base_model(gateway: GatewayServer, model: str) -> str | None: pass # Check if it's directly a supported base model - if model in gateway._supported_model_names: + if model in gateway.supported_model_names: return model # Fallback: if there's exactly one supported model, use it - if len(gateway._supported_model_names) == 1: - return next(iter(gateway._supported_model_names)) + if len(gateway.supported_model_names) == 1: + return next(iter(gateway.supported_model_names)) return None @@ -211,10 +212,6 @@ def _build_sticky_headers(sticky_key: str, request: Request) -> dict[str, str]: return build_routing_headers(sticky_key, auth) -# Per-process cache; each Ray Serve worker holds its own instance. -_template_initialized: set[str] = set() - - async def _ensure_template( gateway: GatewayServer, base_model: str, @@ -223,10 +220,10 @@ async def _ensure_template( ) -> None: """Ensure the sampler has a chat template set for encoding Trajectory inputs. - Called once per base_model (cached in-process). On failure, logs a warning - but doesn't block — the sampler will return its own error if needed. + Called once per base_model (cached on the ``GatewayServer`` instance). On failure, + logs a warning but doesn't block -- the sampler will return its own error if needed. """ - if base_model in _template_initialized: + if base_model in gateway._template_initialized: return template_cls = get_template_for_model(base_model) @@ -239,14 +236,14 @@ async def _ensure_template( try: resp = await gateway.proxy.proxy_request( request, - endpoint='twinkle/set_template', + endpoint=routes.TWINKLE_SET_TEMPLATE, base_model=base_model, service_type='sampler', body_override=set_template_body, extra_headers=sticky_headers, ) if resp.status_code == 200: - _template_initialized.add(base_model) + gateway._template_initialized.add(base_model) else: logger.warning('set_template failed: %s', resp.body.decode()[:200]) except Exception as e: diff --git a/src/twinkle/server/gateway/proxy.py b/src/twinkle/server/gateway/proxy.py index 3177a51c..c12c5bc4 100644 --- a/src/twinkle/server/gateway/proxy.py +++ b/src/twinkle/server/gateway/proxy.py @@ -17,6 +17,7 @@ from twinkle.utils.logger import get_logger from twinkle_client.http.headers import H_MULTIPLEX, H_MULTIPLEX_LEGACY, H_REQUEST_ID, H_REQUEST_ID_LEGACY from twinkle_client.types.errors import ErrorCategory, ErrorPayload +from . import routes logger = get_logger() @@ -58,7 +59,6 @@ def _build_target_url(self, service_type: str, base_model: str, endpoint: str) - Returns: Complete target URL for the internal service """ - prefix = self.route_prefix.rstrip('/') if self.route_prefix else '' host = self.http_options.get('host', 'localhost') port = self.http_options.get('port', 8000) @@ -66,7 +66,7 @@ def _build_target_url(self, service_type: str, base_model: str, endpoint: str) - host = 'localhost' base_url = f'http://{host}:{port}' - return f'{base_url}{prefix}/{service_type}/{base_model}/{endpoint}' + return f'{base_url}{routes.target_url(self.route_prefix, service_type, base_model, endpoint)}' def _prepare_headers(self, request_headers) -> dict[str, str]: """Prepare headers for proxying by removing problematic headers.""" @@ -217,7 +217,7 @@ async def proxy_to_model(self, request: Request, endpoint: str, base_model: str) endpoint: The tinker endpoint name (e.g., 'create_model', 'forward') base_model: The base model name for routing """ - return await self.proxy_request(request, f'tinker/{endpoint}', base_model, 'model') + return await self.proxy_request(request, routes.tinker_endpoint(endpoint), base_model, 'model') async def proxy_to_sampler(self, request: Request, endpoint: str, base_model: str) -> Response: """Proxy request to sampler's tinker endpoint (/tinker/). @@ -227,4 +227,4 @@ async def proxy_to_sampler(self, request: Request, endpoint: str, base_model: st endpoint: The tinker endpoint name (e.g., 'asample') base_model: The base model name for routing """ - return await self.proxy_request(request, f'tinker/{endpoint}', base_model, 'sampler') + return await self.proxy_request(request, routes.tinker_endpoint(endpoint), base_model, 'sampler') diff --git a/src/twinkle/server/gateway/routes.py b/src/twinkle/server/gateway/routes.py new file mode 100644 index 00000000..0cececc0 --- /dev/null +++ b/src/twinkle/server/gateway/routes.py @@ -0,0 +1,26 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Route-path constants shared by the gateway's proxy and its OpenAI bridge. + +Single source for the ``route_prefix`` convention these three places must agree on; they +previously agreed by having the same string typed out in three files. Placed on +the consumer side (gateway) rather than in ``launcher`` -- launcher *produces* the +``route_prefix``, gateway consumes it -- to avoid a new ``gateway -> launcher`` edge. +""" +from __future__ import annotations + +# Downstream endpoint paths the gateway proxies to (relative to the service's route prefix). +TWINKLE_SAMPLE = 'twinkle/sample' +TWINKLE_SAMPLE_STREAM = 'twinkle/sample_stream' +TWINKLE_SET_TEMPLATE = 'twinkle/set_template' + +TINKER_PREFIX = 'tinker' + + +def tinker_endpoint(endpoint: str) -> str: + """``tinker/`` -- the shape ``proxy_to_model`` / ``proxy_to_sampler`` build.""" + return f'{TINKER_PREFIX}/{endpoint}' + + +def target_url(route_prefix: str, service_type: str, base_model: str, endpoint: str) -> str: + """The one definition of ``{route_prefix}/{service_type}/{base_model}/{endpoint}``.""" + return f'{route_prefix.rstrip("/")}/{service_type}/{base_model}/{endpoint}' diff --git a/src/twinkle/server/gateway/tinker_handlers.py b/src/twinkle/server/gateway/tinker_handlers.py index 184023c1..a2dfd0dd 100644 --- a/src/twinkle/server/gateway/tinker_handlers.py +++ b/src/twinkle/server/gateway/tinker_handlers.py @@ -2,7 +2,7 @@ """ Tinker-compatible gateway handlers. -All endpoints are prefixed /* and registered via _register_tinker_routes(app, self_fn). +All endpoints are prefixed /* and registered via _register_gateway_tinker_routes(app, self_fn). self_fn is injected via FastAPI Depends to obtain the GatewayServer instance at request time. """ from __future__ import annotations @@ -17,14 +17,13 @@ from twinkle.hub import HubOperation from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager +from twinkle.server.middleware.auth import get_token_from_request from twinkle.server.state.models import FutureFailureRecord -from twinkle.server.utils.auth import get_token_from_request -from twinkle.server.utils.task_errors import trim_traceback +from twinkle.server.task_errors import trim_traceback from twinkle.utils.logger import get_logger -from .services import create_session as create_session_use_case -from .services import delete_checkpoint -from .services import get_training_run as get_training_run_use_case -from .services import get_weights_info, list_checkpoints, list_training_runs, poll_future, touch_session +from .use_cases import delete_checkpoint +from .use_cases import get_training_run as get_training_run_use_case +from .use_cases import get_weights_info, list_checkpoints, list_training_runs, poll_future logger = get_logger() @@ -66,7 +65,7 @@ def _tinker_error_from_failure(stored: Any, *, request_id: str) -> dict[str, Any return payload -def _register_tinker_routes(app: FastAPI, self_fn: Callable[[], GatewayServer]) -> None: +def _register_gateway_tinker_routes(app: FastAPI, self_fn: Callable[[], GatewayServer]) -> None: """Register all /* Tinker routes on the given FastAPI app. self_fn is a zero-argument callable that returns the current GatewayServer @@ -97,14 +96,14 @@ async def create_session( body: types.CreateSessionRequest, self: GatewayServer = Depends(self_fn), ) -> types.CreateSessionResponse: - session_id = await create_session_use_case(self.state, body.model_dump()) + session_id = await self.state.create_session(body.model_dump()) return types.CreateSessionResponse(session_id=session_id) @app.post('/session_heartbeat') async def session_heartbeat( request: Request, body: types.SessionHeartbeatRequest, self: GatewayServer = Depends(self_fn) ) -> types.SessionHeartbeatResponse: # noqa: E125 - alive = await touch_session(self.state, body.session_id) + alive = await self.state.touch_session(body.session_id) if not alive: raise HTTPException(status_code=404, detail='Unknown session') return types.SessionHeartbeatResponse() diff --git a/src/twinkle/server/gateway/twinkle_handlers.py b/src/twinkle/server/gateway/twinkle_handlers.py index 83aa84d9..a78f7730 100644 --- a/src/twinkle/server/gateway/twinkle_handlers.py +++ b/src/twinkle/server/gateway/twinkle_handlers.py @@ -2,7 +2,7 @@ """ Twinkle-native gateway handlers. -All endpoints are prefixed /twinkle/* and registered via _register_twinkle_routes(app, self_fn). +All endpoints are prefixed /twinkle/* and registered via _register_gateway_twinkle_routes(app, self_fn). """ from __future__ import annotations @@ -18,17 +18,16 @@ from twinkle.server.exceptions import RequestRejectedError, ResourceNotFoundError from twinkle.server.lifecycle.envelope import envelope_from_record from twinkle.server.lifecycle.poll_config import long_poll_window -from twinkle.server.utils.auth import get_token_from_request +from twinkle.server.middleware.auth import get_token_from_request from twinkle.utils.logger import get_logger -from .services import create_session as create_session_use_case -from .services import delete_checkpoint -from .services import get_training_run as get_training_run_use_case -from .services import get_weights_info, list_checkpoints, list_training_runs, poll_future, touch_session +from .use_cases import delete_checkpoint +from .use_cases import get_training_run as get_training_run_use_case +from .use_cases import get_weights_info, list_checkpoints, list_training_runs, poll_future logger = get_logger() -def _register_twinkle_routes(app: FastAPI, self_fn: Callable[[], GatewayServer]) -> None: +def _register_gateway_twinkle_routes(app: FastAPI, self_fn: Callable[[], GatewayServer]) -> None: """Register all /twinkle/* routes on the given FastAPI app.""" @app.get('/twinkle/capacity_info', response_model=types.CapacityInfoResponse) @@ -105,7 +104,7 @@ async def create_session( body: types.CreateSessionRequest, self: GatewayServer = Depends(self_fn), ) -> types.CreateSessionResponse: - session_id = await create_session_use_case(self.state, body.model_dump()) + session_id = await self.state.create_session(body.model_dump()) return types.CreateSessionResponse(session_id=session_id) @app.post('/twinkle/session_heartbeat', response_model=types.SessionHeartbeatResponse) @@ -114,7 +113,7 @@ async def session_heartbeat( body: types.SessionHeartbeatRequest, self: GatewayServer = Depends(self_fn), ) -> types.SessionHeartbeatResponse: - alive = await touch_session(self.state, body.session_id) + alive = await self.state.touch_session(body.session_id) if not alive: raise ResourceNotFoundError('Unknown session') return types.SessionHeartbeatResponse() diff --git a/src/twinkle/server/gateway/services.py b/src/twinkle/server/gateway/use_cases.py similarity index 78% rename from src/twinkle/server/gateway/services.py rename to src/twinkle/server/gateway/use_cases.py index b4f79f75..26691014 100644 --- a/src/twinkle/server/gateway/services.py +++ b/src/twinkle/server/gateway/use_cases.py @@ -1,5 +1,14 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Protocol-neutral gateway use cases shared by Tinker and Twinkle adapters.""" +"""Gateway use cases that do real assembly or control flow. + +What is *not* here is the point: ``create_session`` / ``touch_session`` were one-line +forwards to ``state``, so "is it in this file?" told a reader nothing. Now the file holds +only the long-poll loop and the five checkpoint use cases, which wire up +``create_*_manager(token, client_type)`` -- i.e. things a handler cannot express in one +line. The four handlers that call ``self.state`` directly (``get_capacity_info``, +``cancel_future``, ``get_cleanup_stats``, ``get_model_metadata``) deliberately stay +direct: wrapping them for symmetry would add forwarding, not structure. +""" from __future__ import annotations import asyncio @@ -18,14 +27,6 @@ class FuturePollResult: timed_out: bool -async def create_session(state: Any, metadata: dict[str, Any]) -> str: - return await state.create_session(metadata) - - -async def touch_session(state: Any, session_id: str) -> bool: - return await state.touch_session(session_id) - - async def poll_future(state: Any, request_id: str) -> FuturePollResult: """Long-poll one canonical future record without constructing wire responses.""" deadline = asyncio.get_running_loop().time() + long_poll_window() diff --git a/src/twinkle/server/launcher/server_launcher.py b/src/twinkle/server/launcher/server_launcher.py index 99ceaa74..75fd0bd5 100644 --- a/src/twinkle/server/launcher/server_launcher.py +++ b/src/twinkle/server/launcher/server_launcher.py @@ -18,9 +18,9 @@ from twinkle import get_logger from twinkle.hub.model_alias import MODEL_ID_ALIASES_ENV, build_model_alias_map +from twinkle.patch.ray_serve import apply_ray_serve_patches, get_runtime_env_for_patches from twinkle.server.config import ServerConfig from twinkle.server.config.application_spec import ApplicationSpec -from twinkle.server.utils.ray_serve_patch import apply_ray_serve_patches, get_runtime_env_for_patches from .builder_registry import get_builders, resolve_builder from .env_propagation import build_propagated_env_vars diff --git a/src/twinkle/server/lifecycle/envelope.py b/src/twinkle/server/lifecycle/envelope.py index 6a97b3f7..85274084 100644 --- a/src/twinkle/server/lifecycle/envelope.py +++ b/src/twinkle/server/lifecycle/envelope.py @@ -11,7 +11,7 @@ from typing import Any from twinkle.server.state.models import FutureFailureRecord -from twinkle.server.utils.task_errors import trim_traceback +from twinkle.server.task_errors import trim_traceback from twinkle_client.types.errors import ErrorCategory, ErrorPayload from twinkle_client.types.lifecycle import TaskEnvelope diff --git a/src/twinkle/server/lifecycle/protocols.py b/src/twinkle/server/lifecycle/protocols.py new file mode 100644 index 00000000..c05ea12b --- /dev/null +++ b/src/twinkle/server/lifecycle/protocols.py @@ -0,0 +1,60 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Static declarations of what a deployment class must provide. + +Not a new layer and not a base class: these are ``typing.Protocol`` declarations that +turn the implicit host contract of ``run_submit`` / ``input_metrics`` / the queue mixins +into something a type checker can check. Today those requirements are satisfied by duck +typing and ``getattr`` fallbacks, so a missing attribute surfaces at run time -- sometimes +inside a coroutine sitting in the compute queue. + +Deliberately two layers, matching the two real shapes: every queued deployment +(Gateway/Model/Sampler/Processor) satisfies ``QueuedDeployment``; only ``ModelManagement`` +satisfies ``DataParallelDeployment`` (it is the one with ``data_world_size``). +``DataPlaneManagement`` satisfies neither -- it has no ``state`` and opts out of the +cleanup middleware via ``attach_cleanup_middleware=False``; the gate below is what makes +that fact visible statically instead of only via that boolean. +""" +from __future__ import annotations + +from fastapi import Request +from typing import Any, Protocol, runtime_checkable + +from twinkle.server.lifecycle.envelope import TaskEnvelope +from twinkle.server.state import ServerState +from twinkle.server.task_queue.config import TaskQueueConfig + + +@runtime_checkable +class QueuedDeployment(Protocol): + """A deployment that admits requests through the compute queue.""" + + state: ServerState + replica_id: str + + @property + def task_queue_config(self) -> TaskQueueConfig: + ... + + async def _on_request_start(self, request: Request) -> str: + ... + + def assert_resource_exists(self, resource_id: str | None) -> None: + ... + + async def _peek_terminal(self, request_id: str, *, fallback_status: str) -> TaskEnvelope: + ... + + async def submit_and_peek(self, *args: Any, **kwargs: Any) -> TaskEnvelope: + ... + + async def call_backend(self, fn: Any, /, *args: Any, admit: bool = True, **kwargs: Any) -> Any: + ... + + +@runtime_checkable +class DataParallelDeployment(QueuedDeployment, Protocol): + """A queued deployment that also shards a batch across data-parallel ranks.""" + + @property + def data_world_size(self) -> int: + ... diff --git a/src/twinkle/server/lifecycle/submit.py b/src/twinkle/server/lifecycle/submit.py index dc02074e..a69e3157 100644 --- a/src/twinkle/server/lifecycle/submit.py +++ b/src/twinkle/server/lifecycle/submit.py @@ -2,7 +2,7 @@ """Submit_Endpoint shell and the named seams every queued handler shares. The Inline_Fast_Path wait itself (``submit_and_peek``) lives on -:class:`~twinkle.server.utils.task_queue.mixin.TaskQueueMixin`, since it operates on +:class:`~twinkle.server.task_queue.mixin.TaskQueueMixin`, since it operates on queue state; this module owns the request-shaped pieces around it. """ from __future__ import annotations @@ -10,15 +10,18 @@ import uuid from collections.abc import Callable, Coroutine from fastapi import Request -from typing import Any +from typing import TYPE_CHECKING, Any from twinkle.data_format import InputFeature, Trajectory, is_encoded -from twinkle.server.utils.auth import get_session_id_from_request +from twinkle.server.middleware.auth import get_session_id_from_request from twinkle.server.validation import assert_request_supported from twinkle_client.types.base import FieldRole, fields_with_role from twinkle_client.types.data import export_batch from twinkle_client.types.lifecycle import TaskEnvelope +if TYPE_CHECKING: + from twinkle.server.lifecycle.protocols import DataParallelDeployment, QueuedDeployment + # --------------------------------------------------------------------------- # # Named seams shared by every queued twinkle-native handler. # --------------------------------------------------------------------------- # @@ -94,7 +97,7 @@ def backend_kwargs(body: Any) -> dict[str, Any]: return kwargs -def input_metrics(self, body: Any, *, data_parallel: bool = False) -> dict[str, Any]: +def input_metrics(self: 'DataParallelDeployment', body: Any, *, data_parallel: bool = False) -> dict[str, Any]: """Seam C: scheduling metrics (input_tokens, and batch_size/data_world_size). Reads validated wire models, so no isinstance guards: ``inputs`` is a list and @@ -118,7 +121,7 @@ def resolve_twinkle_adapter_name(request: Request, adapter_name: str | None) -> async def run_submit( - self, + self: 'QueuedDeployment', request: Request, body: Any, *, @@ -195,7 +198,7 @@ async def _task(): if seq_id is not None: session_id = get_session_id_from_request(request) or request.state.request_id dedup_key = f'seq::{session_id}::{adapter_name or "-"}::{seq_id}' - ttl = int(self._task_queue_config.effective_execution_timeout) + 60 + ttl = int(self.task_queue_config.effective_execution_timeout) + 60 prior_request_id = await self.state.claim_seq(dedup_key, request_id, ttl) if prior_request_id is not None: return await self._peek_terminal(prior_request_id, fallback_status='pending') diff --git a/src/twinkle/server/middleware/__init__.py b/src/twinkle/server/middleware/__init__.py new file mode 100644 index 00000000..b1f0e5b0 --- /dev/null +++ b/src/twinkle/server/middleware/__init__.py @@ -0,0 +1,6 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""HTTP middleware for the server deployments (moved here from ``server/utils/`` by R14). + +Sits next to ``deployment.py``'s middleware stack: ``auth.verify_request_token`` is the +token-verification middleware registered by ``build_deployment_app``. +""" diff --git a/src/twinkle/server/utils/auth.py b/src/twinkle/server/middleware/auth.py similarity index 100% rename from src/twinkle/server/utils/auth.py rename to src/twinkle/server/middleware/auth.py diff --git a/src/twinkle/server/model/app.py b/src/twinkle/server/model/app.py index dd7c98b2..e55e4a8d 100644 --- a/src/twinkle/server/model/app.py +++ b/src/twinkle/server/model/app.py @@ -14,24 +14,23 @@ from typing import Any from twinkle import DeviceGroup -from twinkle.server.common.router import StickyLoraRequestRouter -from twinkle.server.deployment import LazyCleanupMixin, bind_deployment, build_deployment_app, init_twinkle_runtime +from twinkle.server.config.backend_dispatch import BackendSelector +from twinkle.server.deployment import LazyCleanupMixin, bind_deployment, build_deployment_app from twinkle.server.exceptions import FullModeBusyError +from twinkle.server.middleware.auth import get_token_from_request +from twinkle.server.model.routing import StickyLoraRequestRouter +from twinkle.server.runtime import init_twinkle_runtime +from twinkle.server.session_resource import AdapterManagerMixin from twinkle.server.state import ServerState, get_server_state +from twinkle.server.task_queue import TaskQueueConfig, TaskQueueMixin from twinkle.server.utils import wrap_builder_with_device_group_env -from twinkle.server.utils.auth import get_token_from_request -from twinkle.server.utils.backend_dispatch import BackendSelector -from twinkle.server.utils.session_resource import AdapterManagerMixin -from twinkle.server.utils.task_queue import TaskQueueConfig, TaskQueueMixin from twinkle.utils.logger import get_logger -from .tinker_handlers import _register_tinker_routes -from .twinkle_handlers import _register_twinkle_routes +from .tinker_handlers import _register_model_tinker_routes +from .twinkle_handlers import _register_model_twinkle_routes logger = get_logger() -# ``FullModeBusyError`` lives in ``twinkle.server.exceptions``; re-exported here -# for backwards compatibility with callers importing it from this module. -__all__ = ['FullModeBusyError', 'ModelManagement', 'build_model_app'] +__all__ = ['ModelManagement', 'build_model_app'] # Ctor kwargs consumed by the MultiLora wrappers' signatures but unknown to the # plain (full-parameter) model classes, where **kwargs flows into HF @@ -133,7 +132,7 @@ async def __init__(self, self.data_plane = DataPlaneProxy(data_plane_url) self._replica_registered = False self._model_unhealthy = False - self._health_probe_task = None + self._health_probe_task: asyncio.Task | None = None actors = getattr(self.model, '_actors', None) self._init_task_queue( @@ -143,7 +142,7 @@ async def __init__(self, on_backend_timeout=self._probe_after_timeout, collect_width=len(actors) if actors else 1, ) - self.model._ray_get_timeout = self._task_queue_config.effective_execution_timeout + self.model._ray_get_timeout = self.task_queue_config.effective_execution_timeout self._init_adapter_manager(**(adapter_config or {})) await self._register_replica_on_startup() # Note: countdown task is started lazily in _ensure_sticky() @@ -310,8 +309,8 @@ def build_model_app(model_id: str, # teardown via ``on_shutdown`` and its sticky-LoRA router via # ``request_router_config``. def register_routes(app: FastAPI, get_self: Any) -> None: - _register_tinker_routes(app, get_self) - _register_twinkle_routes(app, get_self) + _register_model_tinker_routes(app, get_self) + _register_model_twinkle_routes(app, get_self) async def _on_shutdown(servable: Any) -> None: await servable.shutdown() diff --git a/src/twinkle/server/model/backends/megatron_model.py b/src/twinkle/server/model/backends/megatron_model.py index 15dd3470..b0622824 100644 --- a/src/twinkle/server/model/backends/megatron_model.py +++ b/src/twinkle/server/model/backends/megatron_model.py @@ -16,9 +16,9 @@ from twinkle.data_format import InputFeature, Trajectory from twinkle.infra import collect_tensor_dict from twinkle.model.megatron import MegatronModel, MultiLoraMegatronModel -from twinkle.server.common.datum import datum_to_input_feature, extract_rl_features_for_loss from twinkle.server.model.backends.common import (TwinkleCompatModelBase, clean_metrics, collect_forward_backward_results, to_cpu_safe_output) +from twinkle.server.model.tinker_datum import datum_to_input_feature, extract_rl_features_for_loss from twinkle.utils.nccl_safe import nccl_safe_megatron @@ -102,18 +102,16 @@ def tinker_calculate_metric(self, is_training, **kwargs): return clean_metrics(metric) @remote_function(dispatch='all', sync=True, timeout=3600) - def tinker_load(self, checkpoint_dir: str, **kwargs): - """Load checkpoint with token-based isolation support.""" - token = kwargs.pop('token', None) - if not token: - raise ValueError('Token is required for loading checkpoints') - from twinkle.server.checkpoint import create_checkpoint_manager - checkpoint_manager = create_checkpoint_manager(token, client_type='tinker') - resolved = checkpoint_manager.resolve_load_path(checkpoint_dir) - if resolved.is_twinkle_path: - return super().load(name=resolved.checkpoint_name, output_dir=resolved.checkpoint_dir, **kwargs) - else: - return super().load(name=resolved.checkpoint_name, **kwargs) + def tinker_load(self, *, checkpoint_name: str, output_dir: str | None = None, **kwargs): + """Load a checkpoint from an already-resolved location. + + Path resolution (token isolation, twinkle-vs-external path shapes) belongs to the + handler layer: it is a server storage policy, and this class runs inside a Ray + actor as a compute backend. + """ + if output_dir is not None: + return super().load(name=checkpoint_name, output_dir=output_dir, **kwargs) + return super().load(name=checkpoint_name, **kwargs) # ------------------------------------------------------------------ # Twinkle-native methods (InputFeature/Trajectory-based I/O) diff --git a/src/twinkle/server/model/backends/mock_model.py b/src/twinkle/server/model/backends/mock_model.py index c9bc12ff..d204090a 100644 --- a/src/twinkle/server/model/backends/mock_model.py +++ b/src/twinkle/server/model/backends/mock_model.py @@ -140,7 +140,7 @@ def calculate_metric(self, *args: Any, **kwargs: Any) -> dict[str, float]: return {'loss': 0.5, 'grad_norm': 0.1} @remote_function() - def tinker_load(self, checkpoint_dir: str, **kwargs: Any) -> None: + def tinker_load(self, *, checkpoint_name: str, output_dir: str | None = None, **kwargs: Any) -> None: return None # ----- Configuration setters ----------------------------------------- # diff --git a/src/twinkle/server/model/backends/transformers_model.py b/src/twinkle/server/model/backends/transformers_model.py index ff709a69..c36e53c8 100644 --- a/src/twinkle/server/model/backends/transformers_model.py +++ b/src/twinkle/server/model/backends/transformers_model.py @@ -19,9 +19,9 @@ from twinkle.infra import collect_tensor_dict from twinkle.model import MultiLoraTransformersModel from twinkle.model.transformers import TransformersModel -from twinkle.server.common.datum import datum_to_input_feature, extract_rl_features_for_loss from twinkle.server.model.backends.common import (TwinkleCompatModelBase, clean_metrics, collect_forward_backward_results, to_cpu_safe_output) +from twinkle.server.model.tinker_datum import datum_to_input_feature, extract_rl_features_for_loss class _TransformersTinkerCompatMixin(TwinkleCompatModelBase): @@ -80,18 +80,18 @@ def tinker_calculate_metric(self, is_training, **kwargs): return clean_metrics(metric) @remote_function() - def tinker_load(self, checkpoint_dir: str, **kwargs): - """Load checkpoint with token-based isolation support.""" - token = kwargs.pop('token', None) - if not token: - raise ValueError('Token is required for loading checkpoints') - from twinkle.server.checkpoint import create_checkpoint_manager - checkpoint_manager = create_checkpoint_manager(token, client_type='tinker') - resolved = checkpoint_manager.resolve_load_path(checkpoint_dir) - if resolved.is_twinkle_path: - return super().load(name=resolved.checkpoint_name, output_dir=resolved.checkpoint_dir, **kwargs) - else: - return super().load(name=resolved.checkpoint_name, **kwargs) + def tinker_load(self, *, checkpoint_name: str, output_dir: str | None = None, **kwargs): + """Load a checkpoint from an already-resolved location. + + Path resolution (token isolation, twinkle-vs-external path shapes) belongs to the + handler layer: it is a server storage policy, and this class runs inside a Ray + actor as a compute backend. Keeping it here meant a checkpoint-layout change had + to touch GPU-side code -- the hardest layer to test -- and the resolution block + was duplicated verbatim in the megatron backend. + """ + if output_dir is not None: + return super().load(name=checkpoint_name, output_dir=output_dir, **kwargs) + return super().load(name=checkpoint_name, **kwargs) # ------------------------------------------------------------------ # Twinkle-native methods (InputFeature/Trajectory-based I/O) diff --git a/src/twinkle/server/common/router.py b/src/twinkle/server/model/routing.py similarity index 98% rename from src/twinkle/server/common/router.py rename to src/twinkle/server/model/routing.py index 0aabc63b..02b7669a 100644 --- a/src/twinkle/server/common/router.py +++ b/src/twinkle/server/model/routing.py @@ -1,5 +1,4 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -# Moved from tinker/common/router.py — logic unchanged. from ray.serve.request_router import FIFOMixin, MultiplexMixin, PendingRequest, ReplicaID, RequestRouter, RunningReplica from twinkle.server.state import ServerState, get_server_state diff --git a/src/twinkle/server/common/datum.py b/src/twinkle/server/model/tinker_datum.py similarity index 100% rename from src/twinkle/server/common/datum.py rename to src/twinkle/server/model/tinker_datum.py diff --git a/src/twinkle/server/model/tinker_handlers.py b/src/twinkle/server/model/tinker_handlers.py index d868a0d2..5eca201b 100644 --- a/src/twinkle/server/model/tinker_handlers.py +++ b/src/twinkle/server/model/tinker_handlers.py @@ -1,9 +1,11 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -""" -Tinker-compatible model handler mixin. +"""Tinker-compatible routes for the Model deployment. -All endpoints are prefixed /tinker/... and use schedule_task() returning UntypedAPIFuture. -self_fn is injected via FastAPI Depends to obtain the ModelManagement instance at request time. +Registered by ``_register_model_tinker_routes(app, self_fn)`` -- module-level route +registration closing over ``self_fn`` via ``Depends``, not a mixin: there is no +inheritance relationship with the deployment class. All endpoints are prefixed +/tinker/... and use schedule_task() returning UntypedAPIFuture. ``self_fn`` is injected +via FastAPI Depends to obtain the ModelManagement instance at request time. """ from __future__ import annotations @@ -18,14 +20,14 @@ from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager from twinkle.server.exceptions import FullModeBusyError +from twinkle.server.task_queue.types import UserTaskError from twinkle.server.utils import get_template_for_model -from twinkle.server.utils.task_queue.types import UserTaskError from twinkle.utils.logger import get_logger logger = get_logger() -def _register_tinker_routes(app: FastAPI, self_fn: Callable[[], ModelManagement]) -> None: +def _register_model_tinker_routes(app: FastAPI, self_fn: Callable[[], ModelManagement]) -> None: """Register all /tinker/* routes on the given FastAPI app. self_fn is a zero-argument callable that returns the current ModelManagement @@ -330,12 +332,17 @@ async def _do_load(): assert self.model is not None, 'Model not loaded, please load model first' adapter_name = self.get_adapter_name(adapter_name=body.model_id) self.assert_resource_exists(adapter_name) + # Path resolution (token isolation, twinkle-vs-external path shapes) is a + # server storage policy and belongs in the handler, not in the GPU-side + # backend actor. The backend receives an already-resolved location. + checkpoint_manager = create_checkpoint_manager(token, client_type='tinker') + resolved = checkpoint_manager.resolve_load_path(body.path) await self.call_backend( self.model.tinker_load, - checkpoint_dir=body.path, + checkpoint_name=resolved.checkpoint_name, + output_dir=resolved.checkpoint_dir if resolved.is_twinkle_path else None, load_optimizer=body.optimizer, - adapter_name=self.resolve_model_adapter_name(adapter_name), - token=token) + adapter_name=self.resolve_model_adapter_name(adapter_name)) self.set_resource_state(adapter_name, 'grad_ready', False) return types.LoadWeightsResponse(path=body.path, type='load_weights') except Exception: diff --git a/src/twinkle/server/model/twinkle_handlers.py b/src/twinkle/server/model/twinkle_handlers.py index c20665f5..adc548f2 100644 --- a/src/twinkle/server/model/twinkle_handlers.py +++ b/src/twinkle/server/model/twinkle_handlers.py @@ -1,12 +1,13 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -""" -Twinkle-native model handler mixin. - -All queued endpoints are prefixed /twinkle/... and return a Task_Envelope via the -shared ``run_submit`` judgment sequence: the handler submits work and returns -immediately, and the client's Client_Future_Layer resolves the envelope to a -terminal state. self_fn is injected via FastAPI Depends to obtain the -ModelManagement instance at request time. +"""Twinkle-native routes for the Model deployment. + +Registered by ``_register_model_twinkle_routes(app, self_fn)`` -- module-level route +registration closing over ``self_fn`` via ``Depends``, not a mixin: there is no +inheritance relationship with the deployment class. All queued endpoints are prefixed +/twinkle/... and return a Task_Envelope via the shared ``run_submit`` judgment sequence: +the handler submits work and returns immediately, and the client's Client_Future_Layer +resolves the envelope to a terminal state. ``self_fn`` is injected via FastAPI Depends to +obtain the ModelManagement instance at request time. """ from __future__ import annotations @@ -25,9 +26,9 @@ from twinkle.server.exceptions import RequestRejectedError, TrainModeMismatchError from twinkle.server.lifecycle.submit import (backend_kwargs, input_metrics, resolve_twinkle_adapter_name, run_submit, to_backend_inputs) +from twinkle.server.middleware.auth import get_session_id_from_request from twinkle.server.model.data_plane_inputs import (data_plane_request_shape, merge_forward_kwargs, resolve_data_plane_model_inputs, select_output_rows) -from twinkle.server.utils.auth import get_session_id_from_request from twinkle.server.validation import BackendCapability from twinkle.utils.logger import get_logger @@ -50,7 +51,7 @@ def _data_plane_metrics(self, body): return {'input_tokens': input_tokens, 'batch_size': batch_size, 'data_world_size': self.data_world_size} -def _register_twinkle_routes(app: FastAPI, self_fn: Callable[[], ModelManagement]) -> None: +def _register_model_twinkle_routes(app: FastAPI, self_fn: Callable[[], ModelManagement]) -> None: """Register all /twinkle/* routes on the given FastAPI app. self_fn is a zero-argument callable that returns the current ModelManagement diff --git a/src/twinkle/server/processor/app.py b/src/twinkle/server/processor/app.py index 6b268d2b..b471cede 100644 --- a/src/twinkle/server/processor/app.py +++ b/src/twinkle/server/processor/app.py @@ -18,11 +18,11 @@ from ray import serve from typing import Any -import twinkle -from twinkle import DeviceGroup, DeviceMesh, get_logger +from twinkle import DeviceGroup, get_logger from twinkle.server.deployment import LazyCleanupMixin, bind_deployment, build_deployment_app +from twinkle.server.runtime import init_twinkle_runtime +from twinkle.server.session_resource import ProcessorManagerMixin from twinkle.server.state import ServerState, get_server_state -from twinkle.server.utils.session_resource import ProcessorManagerMixin from .twinkle_handlers import _register_processor_routes logger = get_logger() @@ -48,16 +48,13 @@ def __init__(self, nproc_per_node: int = 1, processor_config: dict[str, Any] | None = None): self.device_group = DeviceGroup(**device_group) - twinkle.initialize( - mode='ray', + self.device_mesh = init_twinkle_runtime( + is_mock=False, nproc_per_node=nproc_per_node, - groups=[self.device_group], - lazy_collect=False, - ncpu_proc_per_node=ncpu_proc_per_node) - if 'mesh_dim_names' in device_mesh: - self.device_mesh = DeviceMesh(**device_mesh) - else: - self.device_mesh = DeviceMesh.from_sizes(**device_mesh) + device_group=self.device_group, + device_mesh_dict=device_mesh, + ncpu_proc_per_node=ncpu_proc_per_node, + ) # processor objects keyed by processor_id self.resource_dict: dict[str, Any] = {} diff --git a/src/twinkle/server/processor/twinkle_handlers.py b/src/twinkle/server/processor/twinkle_handlers.py index 222c1dfe..70645f9f 100644 --- a/src/twinkle/server/processor/twinkle_handlers.py +++ b/src/twinkle/server/processor/twinkle_handlers.py @@ -1,10 +1,11 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -""" -Processor management handler mixin. +"""Processor management routes for the Processor deployment. -All endpoints are prefixed /twinkle/... and handle processor lifecycle -(create, call). self_fn is injected via FastAPI Depends to obtain the -ProcessorManagement instance at request time. +Registered by ``_register_processor_routes(app, self_fn)`` -- module-level route +registration closing over ``self_fn`` via ``Depends``, not a mixin: there is no +inheritance relationship with the deployment class. All endpoints are prefixed +/twinkle/... and handle processor lifecycle (create, call). ``self_fn`` is injected via +FastAPI Depends to obtain the ProcessorManagement instance at request time. """ from __future__ import annotations @@ -19,9 +20,9 @@ from .app import ProcessorManagement import twinkle_client.types as types +from twinkle.server.middleware.auth import get_session_id_from_request, get_token_from_request from twinkle.server.telemetry.correlation import SESSION_ID, TOKEN_ID from twinkle.server.telemetry.tracing import traced_operation -from twinkle.server.utils.auth import get_session_id_from_request, get_token_from_request from twinkle.utils.logger import get_logger logger = get_logger() @@ -92,7 +93,8 @@ def _do_create(): # Span the primary processor.create op with token + session correlation. with traced_operation( - f'processor.create.{processor_type_name}.{class_type}', attrs={ + f'processor.create.{processor_type_name}.{class_type}', + attrs={ TOKEN_ID: token, SESSION_ID: session_id, }): @@ -108,8 +110,8 @@ def _do_create(): lease_seconds=self._processor_quota_lease_seconds, ) except Exception as release_error: - logger.warning('Failed to release processor quota after create failure for %s: %r', - processor_id, release_error) + logger.warning('Failed to release processor quota after create failure for %s: %r', processor_id, + release_error) raise return types.ProcessorCreateResponse(processor_id='pid:' + processor_id) diff --git a/src/twinkle/server/runtime.py b/src/twinkle/server/runtime.py new file mode 100644 index 00000000..1a6d8e13 --- /dev/null +++ b/src/twinkle/server/runtime.py @@ -0,0 +1,52 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Twinkle distributed-runtime initialisation for the deployment classes. + +Moved out of ``deployment.py``: that module is deployment-*construction* +infrastructure -- the FastAPI scaffold, the middleware stack, the ``serve.ingress`` +chain -- and ``twinkle.initialize`` + ``DeviceMesh`` construction is neither. It lived +there only because Model and Sampler both needed it; now Processor reuses it too, so its +formerly-inlined copy is gone. +""" +from __future__ import annotations + +from typing import Any + + +def init_twinkle_runtime( + is_mock: bool, + nproc_per_node: int, + device_group: Any, + device_mesh_dict: dict[str, Any], + *, + ncpu_proc_per_node: int | None = None, +) -> Any | None: + """Initialize the Twinkle distributed runtime and build a DeviceMesh. + + Shared by ModelManagement, SamplerManagement and ProcessorManagement ``__init__``. + Returns ``None`` for mock backends (CPU-only, no device mesh). + + ``ncpu_proc_per_node`` is forwarded only when provided; Model/Sampler leave it unset + (preserving their prior behaviour), while Processor passes its own value -- reusing + this function without that parameter would have silently changed the processor's CPU + process count. + """ + import twinkle + from twinkle import DeviceMesh + + if is_mock: + twinkle.initialize( + mode='ray', nproc_per_node=nproc_per_node, ncpu_proc_per_node=1, groups=[device_group], lazy_collect=False) + return None + + init_kwargs: dict[str, Any] = { + 'mode': 'ray', + 'nproc_per_node': nproc_per_node, + 'groups': [device_group], + 'lazy_collect': False, + } + if ncpu_proc_per_node is not None: + init_kwargs['ncpu_proc_per_node'] = ncpu_proc_per_node + twinkle.initialize(**init_kwargs) + if 'mesh_dim_names' in device_mesh_dict: + return DeviceMesh(**device_mesh_dict) + return DeviceMesh.from_sizes(**device_mesh_dict) diff --git a/src/twinkle/server/sampler/app.py b/src/twinkle/server/sampler/app.py index 3d7e92a8..b10c5894 100644 --- a/src/twinkle/server/sampler/app.py +++ b/src/twinkle/server/sampler/app.py @@ -12,12 +12,14 @@ from typing import Any from twinkle import DeviceGroup -from twinkle.server.deployment import LazyCleanupMixin, bind_deployment, build_deployment_app, init_twinkle_runtime +from twinkle.server.config.backend_dispatch import BackendSelector +from twinkle.server.deployment import LazyCleanupMixin, bind_deployment, build_deployment_app +from twinkle.server.middleware.auth import get_token_from_request +from twinkle.server.runtime import init_twinkle_runtime from twinkle.server.state import ServerState, get_server_state +from twinkle.server.task_queue.config import TaskQueueConfig +from twinkle.server.task_queue.mixin import TaskQueueMixin from twinkle.server.utils import wrap_builder_with_device_group_env -from twinkle.server.utils.auth import get_token_from_request -from twinkle.server.utils.backend_dispatch import BackendSelector -from twinkle.server.utils.task_queue import TaskQueueConfig, TaskQueueMixin from twinkle.utils.logger import get_logger from .tinker_handlers import _register_tinker_sampler_routes from .twinkle_handlers import _register_twinkle_sampler_routes @@ -125,7 +127,7 @@ def __init__(self, deployment_name='Sampler', collect_width=len(actors) if actors else 1, ) - self.sampler._ray_get_timeout = self._task_queue_config.effective_execution_timeout + self.sampler._ray_get_timeout = self.task_queue_config.effective_execution_timeout async def shutdown(self) -> None: try: diff --git a/src/twinkle/server/sampler/tinker_handlers.py b/src/twinkle/server/sampler/tinker_handlers.py index 927602b8..153a3939 100644 --- a/src/twinkle/server/sampler/tinker_handlers.py +++ b/src/twinkle/server/sampler/tinker_handlers.py @@ -1,8 +1,10 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -""" -Tinker-compatible sampler handler mixin. +"""Tinker-compatible routes for the Sampler deployment. -Provides POST /tinker/asample using schedule_task() returning UntypedAPIFuture. +Registered by ``_register_tinker_sampler_routes(app, self_fn)`` -- module-level route +registration closing over ``self_fn`` via ``Depends``, not a mixin: there is no +inheritance relationship with the deployment class. Provides POST /tinker/asample using +schedule_task() returning UntypedAPIFuture. """ from __future__ import annotations @@ -19,8 +21,8 @@ from twinkle.data_format import SamplingParams from twinkle.server.checkpoint import create_checkpoint_manager from twinkle.server.sampler.weights import resolve_sampler_weights +from twinkle.server.task_queue.types import UserTaskError from twinkle.server.utils import get_template_for_model -from twinkle.server.utils.task_queue.types import UserTaskError from twinkle.utils.logger import get_logger logger = get_logger() diff --git a/src/twinkle/server/sampler/twinkle_handlers.py b/src/twinkle/server/sampler/twinkle_handlers.py index f44ce57f..de8c9ffb 100644 --- a/src/twinkle/server/sampler/twinkle_handlers.py +++ b/src/twinkle/server/sampler/twinkle_handlers.py @@ -1,8 +1,10 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -""" -Twinkle-native sampler handler mixin. +"""Twinkle-native routes for the Sampler deployment. -Provides /twinkle/* sampler endpoints. +Registered by ``_register_twinkle_sampler_routes(app, self_fn)`` -- module-level route +registration closing over ``self_fn`` via ``Depends``, not a mixin: there is no +inheritance relationship with the deployment class. Provides /twinkle/* sampler +endpoints. """ from __future__ import annotations @@ -25,9 +27,9 @@ from twinkle.server.exceptions import EndpointUnavailableError, RequestRejectedError from twinkle.server.lifecycle.submit import backend_kwargs, resolve_twinkle_adapter_name, to_backend_inputs from twinkle.server.sampler.weights import resolve_sampler_weights +from twinkle.server.task_errors import task_error_payload from twinkle.server.telemetry.correlation import MODEL_ID from twinkle.server.telemetry.tracing import traced_operation -from twinkle.server.utils.task_errors import task_error_payload from twinkle.utils.logger import get_logger from twinkle_client.common.json_utils import json_safe from twinkle_client.types import sampler as sampler_types @@ -53,11 +55,6 @@ def _serialize_input_feature(feature: dict) -> dict: return result -def _get_twinkle_sampler_adapter_name(request: Request, adapter_name: str | None) -> str | None: - """Per-session adapter name; delegates to the shared lifecycle resolver.""" - return resolve_twinkle_adapter_name(request, adapter_name) - - def _build_rollout_rows_and_tags( sample_models: list[types.SampleResponseModel], *, @@ -234,7 +231,7 @@ async def _task(): # Resolve adapter adapter_path = None adapter_name = body.adapter_name or '' - full_adapter_name = _get_twinkle_sampler_adapter_name(request, adapter_name) or '' + full_adapter_name = resolve_twinkle_adapter_name(request, adapter_name) or '' if body.adapter_uri: from twinkle.server.checkpoint import create_checkpoint_manager @@ -282,7 +279,7 @@ async def sample_to_data_plane( raise EndpointUnavailableError('sampler_type must be vllm_async') adapter_path = None - full_adapter_name = _get_twinkle_sampler_adapter_name(request, body.adapter_name) or '' + full_adapter_name = resolve_twinkle_adapter_name(request, body.adapter_name) or '' if body.adapter_uri: from twinkle.server.checkpoint import create_checkpoint_manager checkpoint_manager = create_checkpoint_manager(token, client_type='twinkle') @@ -308,8 +305,7 @@ async def _generate_and_store(): adapter_name=full_adapter_name, adapter_path=adapter_path, ) - responses = await _await_generation(self, submission_id, - self._task_queue_config.effective_execution_timeout) + responses = await _await_generation(self, submission_id, self.task_queue_config.effective_execution_timeout) rows, tags = _build_rollout_rows_and_tags( _to_sample_response_models(responses), group_ids=body.group_ids, @@ -365,7 +361,7 @@ async def add_adapter_to_sampler( # would vanish under `python -O`, letting an empty adapter_name reach the backend. if not body.adapter_name: raise RequestRejectedError('`adapter_name` is required and must be non-empty.') - full_adapter_name = _get_twinkle_sampler_adapter_name(request, body.adapter_name) + full_adapter_name = resolve_twinkle_adapter_name(request, body.adapter_name) from peft import LoraConfig config = LoraConfig(**body.config) if isinstance(body.config, dict) else body.config @@ -404,7 +400,7 @@ async def sample_stream( adapter_path = None adapter_name = body.adapter_name or '' - full_adapter_name = _get_twinkle_sampler_adapter_name(request, adapter_name) or '' + full_adapter_name = resolve_twinkle_adapter_name(request, adapter_name) or '' if body.adapter_uri: from twinkle.server.checkpoint import create_checkpoint_manager @@ -453,7 +449,7 @@ async def _no_actor_generator(): q, STREAM_SENTINEL, request_id, - self._task_queue_config.effective_execution_timeout, + self.task_queue_config.effective_execution_timeout, ), media_type='application/x-ndjson', ) diff --git a/src/twinkle/server/utils/session_resource/__init__.py b/src/twinkle/server/session_resource/__init__.py similarity index 100% rename from src/twinkle/server/utils/session_resource/__init__.py rename to src/twinkle/server/session_resource/__init__.py diff --git a/src/twinkle/server/utils/session_resource/adapter.py b/src/twinkle/server/session_resource/adapter.py similarity index 100% rename from src/twinkle/server/utils/session_resource/adapter.py rename to src/twinkle/server/session_resource/adapter.py diff --git a/src/twinkle/server/utils/session_resource/base.py b/src/twinkle/server/session_resource/base.py similarity index 100% rename from src/twinkle/server/utils/session_resource/base.py rename to src/twinkle/server/session_resource/base.py diff --git a/src/twinkle/server/utils/session_resource/processor.py b/src/twinkle/server/session_resource/processor.py similarity index 100% rename from src/twinkle/server/utils/session_resource/processor.py rename to src/twinkle/server/session_resource/processor.py diff --git a/src/twinkle/server/state/__init__.py b/src/twinkle/server/state/__init__.py index 1b6ca9ba..9acc8aca 100644 --- a/src/twinkle/server/state/__init__.py +++ b/src/twinkle/server/state/__init__.py @@ -1,8 +1,9 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -from twinkle.server.config.persistence import PersistenceConfig from .backend import create_backend from .base import BaseManager +from .cleanup_coordinator import ResourceCleanupCoordinator from .config_manager import ConfigManager +from .count_publisher import ResourceCountPublisher from .future_manager import FutureManager from .model_manager import ModelManager from .models import FutureRecord, ModelRecord, SamplingSessionRecord, SessionRecord @@ -28,9 +29,9 @@ # Server state 'ServerState', 'ReplicaRegistry', + 'ResourceCleanupCoordinator', + 'ResourceCountPublisher', 'get_server_state', - 'reset_server_state_cache', # Persistence backend factory - 'PersistenceConfig', 'create_backend', ] diff --git a/src/twinkle/server/state/cleanup_coordinator.py b/src/twinkle/server/state/cleanup_coordinator.py new file mode 100644 index 00000000..c6278c24 --- /dev/null +++ b/src/twinkle/server/state/cleanup_coordinator.py @@ -0,0 +1,247 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Cleanup orchestration and cleanup-leader election. + +Extracted from ``ServerState`` because these ~120 lines need only the +managers' ``count()`` / concrete ``cleanup_expired`` plus the shared +``StateBackend``, yet used to live beside ~25 CRUD facade methods, so any change +here meant re-reading all ~660 lines to confirm nothing else was touched. + +Leader election lives here rather than in ``ServerState`` because it exists *for* +the cleanup loop: exactly one holder of the lease runs the cascade, so four Ray +Serve workers do not each sweep (and do not each publish resource counts). This +module deliberately does NOT import ``telemetry``: the resource-count publishing +is driven through the ``on_become_leader`` / ``on_lose_leader`` callbacks so the +telemetry dependency stays isolated to ``count_publisher``. +""" +from __future__ import annotations + +import asyncio +import functools +import time +import uuid +from collections.abc import Awaitable, Callable, Mapping +from typing import Any + +from twinkle.utils.logger import get_logger +from .backend import StateBackend +from .base import BaseManager + +logger = get_logger() + +# ---------- Cleanup-leader election ------------------------------------------ +# +# Every Ray Serve worker creates its own ``ServerState``; without coordination +# each one would run the periodic cleanup and metrics-publish loop, so a single +# Twinkle deployment would multiply the work and inflate every gauge by the +# worker count. We elect one leader per backend by racing for a TTL-scoped key +# inside the shared StateBackend: the winner runs cleanup + publishes metrics, +# the others stay quiet. + +LEADER_KEY = 'cleanup_leader' # actual backend key: 'cleanup_leader' +LEASE_TTL = 30 # seconds — leader loses the lease after this without a renew +LEASE_RENEW = 10 # seconds — must be < LEASE_TTL/2 so two missed renews still beat the TTL + + +def _renew_if_owner(current: str | None, *, owner: str) -> str | None: + """``update_atomic`` transform: only re-write the lease if it is still mine.""" + if current == owner: + return owner + return None + + +class ResourceCleanupCoordinator: + """Owns the cleanup loop, the cascaded expiry across managers, and leader election.""" + + def __init__( + self, + backend: StateBackend, + managers: Mapping[str, BaseManager], + *, + cleanup_interval: float, + expiration_timeout: float, + sweep_processor_quotas: Callable[[], Awaitable[None]], + on_become_leader: Callable[[], Awaitable[None]] | None = None, + on_lose_leader: Callable[[], Awaitable[None]] | None = None, + ) -> None: + self._backend = backend + self._managers = dict(managers) + self._cleanup_interval = float(cleanup_interval) + self._expiration_timeout = float(expiration_timeout) + self._sweep_processor_quotas = sweep_processor_quotas + self._on_become_leader = on_become_leader + self._on_lose_leader = on_lose_leader + + self._cleanup_task: asyncio.Task | None = None + self._cleanup_running = False + self._leader_id = uuid.uuid4().hex + self._is_leader = False + self._leader_task: asyncio.Task | None = None + self._leader_running = False + + @property + def is_leader(self) -> bool: + return self._is_leader + + # ----- Resource cleanup ----- + + async def cleanup_expired_resources(self) -> dict[str, int]: + """Clean up expired sessions, models, sampling_sessions, and futures. + + Sessions expire based on last_heartbeat (or created_at). Models and sampling + sessions are also cascade-expired when their owning session expires. Futures + expire based on updated_at (or created_at). + """ + current_time = time.time() + cutoff_time = current_time - self._expiration_timeout + + session_mgr = self._managers['sessions'] + model_mgr = self._managers['models'] + sampling_mgr = self._managers['sampling_sessions'] + future_mgr = self._managers['futures'] + + # Determine expired sessions and remove them in a SINGLE pass, then cascade + # the SAME set to dependent resources. Using one authoritative set closes the + # TOCTOU window where a session touched mid-cleanup could survive removal while + # its children were cascade-deleted. + expired_session_ids, sessions_removed = await session_mgr.collect_and_remove_expired(cutoff_time) + + models_removed = await model_mgr.cleanup_expired(cutoff_time, expired_session_ids=expired_session_ids) + samplings_removed = await sampling_mgr.cleanup_expired(cutoff_time, expired_session_ids=expired_session_ids) + + alive_replica_ids = await model_mgr.get_alive_replica_ids(self._expiration_timeout) + futures_removed = await future_mgr.cleanup_expired(cutoff_time, alive_replica_ids=alive_replica_ids) + await self._sweep_processor_quotas() + + return { + 'sessions': sessions_removed, + 'models': models_removed, + 'sampling_sessions': samplings_removed, + 'futures': futures_removed, + } + + async def _cleanup_loop(self) -> None: + """Background task that periodically cleans up expired resources. + + Gated by leader election — non-leader workers skip the actual cleanup so the + same backend isn't swept 4x by 4 deployment workers. + """ + while self._cleanup_running: + try: + await asyncio.sleep(self._cleanup_interval) + if not self._is_leader: + continue + stats = await self.cleanup_expired_resources() + if any(stats.values()): + logger.debug(f'[ServerState Cleanup] Removed expired resources: {stats}') + except asyncio.CancelledError: + break + except Exception as e: + logger.warning(f'[ServerState Cleanup] Error during cleanup: {e}') + continue + + # ----- Leader election ----- + + async def _leader_loop(self) -> None: + """Acquire and renew the cleanup-leader lease every LEASE_RENEW seconds.""" + await self._try_acquire_or_renew() # Race for leadership at startup + while self._leader_running: + try: + await asyncio.sleep(LEASE_RENEW) + await self._try_acquire_or_renew() + except asyncio.CancelledError: + break + except Exception as e: + logger.warning(f'[ServerState Leader] renew error: {e}') + continue + + async def _try_acquire_or_renew(self) -> None: + was_leader = self._is_leader + try: + if self._is_leader: + val = await self._backend.update_atomic( + LEADER_KEY, + functools.partial(_renew_if_owner, owner=self._leader_id), + ttl=LEASE_TTL, + ) + self._is_leader = (val == self._leader_id) + else: + self._is_leader = await self._backend.set_nx(LEADER_KEY, self._leader_id, ttl=LEASE_TTL) + except Exception as e: + logger.warning(f'[ServerState Leader] backend error during election: {e}') + self._is_leader = False + if was_leader: + # Our renewal failed but our lease value may still be sitting in the + # backend, so a plain ``set_nx`` would keep returning False for up to + # LEASE_TTL and leadership would stall unclaimed. Best-effort delete + # ONLY when we were the leader (never steal a lease another replica + # legitimately holds), swallowing errors so a delete failure cannot + # escape the election loop. The next tick can then re-acquire. + try: + await self._backend.delete(LEADER_KEY) + except Exception: + pass + + if self._is_leader and not was_leader: + logger.info(f'[ServerState] became cleanup leader (id={self._leader_id[:8]})') + if self._on_become_leader is not None: + await self._on_become_leader() + elif not self._is_leader and was_leader: + logger.warning(f'[ServerState] lost cleanup leadership (id={self._leader_id[:8]})') + if self._on_lose_leader is not None: + await self._on_lose_leader() + + # ----- Lifecycle ----- + + async def start(self) -> bool: + """Start the background cleanup + leader-election tasks. + + Idempotent: returns ``False`` if already running. The guard lives here (not in + ``ServerState``) so the implementation and its guard can never drift apart into + a double-start. + """ + if self._cleanup_running: + return False + # Rebuild in-memory indexes from backend data before the loops start. + await self._managers['models'].rebuild_indexes() + self._cleanup_running = True + self._cleanup_task = asyncio.create_task(self._cleanup_loop()) + self._leader_running = True + self._leader_task = asyncio.create_task(self._leader_loop()) + return True + + async def stop(self) -> bool: + """Stop the background cleanup + leader-election tasks. Returns ``False`` if not running.""" + if not self._cleanup_running: + return False + self._cleanup_running = False + if self._cleanup_task: + self._cleanup_task.cancel() + self._cleanup_task = None + self._leader_running = False + if self._leader_task: + self._leader_task.cancel() + self._leader_task = None + if self._is_leader: + # Release callback registration; the lease itself expires on its own TTL — + # update_atomic can't express "atomic delete", so we accept a short outage + # where the gauge reads 0 between leaders. + if self._on_lose_leader is not None: + await self._on_lose_leader() + self._is_leader = False + return True + + async def get_cleanup_stats(self) -> dict[str, Any]: + """Get current cleanup configuration and resource counts.""" + return { + 'expiration_timeout': self._expiration_timeout, + 'cleanup_interval': self._cleanup_interval, + 'cleanup_running': self._cleanup_running, + 'is_leader': self._is_leader, + 'leader_id': self._leader_id, + 'resource_counts': { + 'sessions': await self._managers['sessions'].count(), + 'models': await self._managers['models'].count(), + 'sampling_sessions': await self._managers['sampling_sessions'].count(), + 'futures': await self._managers['futures'].count(), + }, + } diff --git a/src/twinkle/server/state/count_publisher.py b/src/twinkle/server/state/count_publisher.py new file mode 100644 index 00000000..aa09c253 --- /dev/null +++ b/src/twinkle/server/state/count_publisher.py @@ -0,0 +1,78 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Resource-count publishing for the cleanup leader. + +This is the single file in ``state/`` that imports the metrics registry. +Pulling ``_metrics_publish_loop`` out of ``ServerState`` shrinks the persistence +layer's dependency on the observability layer to this one module. + +Only the cleanup leader publishes: ``ServerState`` drives this through the +coordinator's ``on_become_leader`` / ``on_lose_leader`` callbacks, so four Ray Serve +workers do not multiply the gauges -- +``test_lgtm_telemetry.py::test_active_sessions_no_4x_inflation`` guards it. +""" +from __future__ import annotations + +import asyncio +from collections.abc import Sequence + +from twinkle.server.telemetry import MetricsRegistry +from twinkle.utils.logger import get_logger +from .base import BaseManager + +logger = get_logger() + + +class ResourceCountPublisher: + """Owns the periodic push of resource counts into the ``MetricsRegistry`` cache. + + The ObservableGauges registered by :class:`MetricsRegistry` read the cache at OTEL + export time and report whatever was pushed last, so this loop is the single writer + of those four gauges. + """ + + def __init__(self, managers: Sequence[tuple[str, BaseManager]], *, interval: float) -> None: + self._managers = tuple(managers) + self._interval = float(interval) + self._task: asyncio.Task | None = None + self._running = False + + async def start(self) -> None: + """Start pushing counts. Idempotent — a second call while running is a no-op.""" + if self._task is not None and not self._task.done(): + return + self._running = True + self._task = asyncio.create_task(self._publish_loop()) + + async def stop(self) -> None: + self._running = False + if self._task is not None: + self._task.cancel() + try: + await self._task + except (asyncio.CancelledError, Exception): + pass + self._task = None + + def clear(self) -> None: + """Zero this worker's resource-gauge cache. + + Called on leadership loss: after the publish loop is cancelled it never + overwrites the cache again, so without zeroing the stale worker would keep + emitting its last counts forever. The new leader publishes the authoritative + counts from its own process. + """ + MetricsRegistry.get().clear_resource_counts() + + async def _publish_loop(self) -> None: + """Push resource counts into the MetricsRegistry cache every N seconds.""" + registry = MetricsRegistry.get() + while self._running: + try: + await asyncio.sleep(self._interval) + for name, mgr in self._managers: + registry.set_resource_count(name, await mgr.count()) + except asyncio.CancelledError: + break + except Exception as e: + logger.debug(f'[ResourceCountPublisher] Error publishing metrics: {e}') + continue diff --git a/src/twinkle/server/state/server_state.py b/src/twinkle/server/state/server_state.py index db443dd3..daadc051 100644 --- a/src/twinkle/server/state/server_state.py +++ b/src/twinkle/server/state/server_state.py @@ -1,7 +1,6 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from __future__ import annotations -import asyncio import functools import math import re @@ -12,14 +11,15 @@ from twinkle.server.config.persistence import PersistenceConfig from twinkle.server.exceptions import ResourceQuotaExceededError -from twinkle.server.telemetry import MetricsRegistry from twinkle.server.telemetry.correlation import (BASE_MODEL, MODEL_ID, REPLICA_ID, SAMPLING_SESSION_ID, SESSION_ID, TOKEN_ID) from twinkle.server.telemetry.tracing import traced_operation from twinkle.utils.logger import get_logger from .backend import StateBackend from .backend.factory import create_backend +from .cleanup_coordinator import ResourceCleanupCoordinator from .config_manager import ConfigManager +from .count_publisher import ResourceCountPublisher from .future_manager import FutureManager from .model_manager import ModelManager from .models import FutureFailureRecord, ModelRecord, SamplingSessionRecord, SessionRecord @@ -28,18 +28,6 @@ logger = get_logger() -# ---------- Cleanup-leader election ------------------------------------------ -# -# Every Ray Serve worker creates its own ``ServerState``; without coordination -# each one would run the periodic cleanup and metrics-publish loop, so a -# single Twinkle deployment would multiply the work and inflate every gauge -# by the worker count. We elect one leader per backend by racing for a TTL- -# scoped key inside the shared StateBackend: the winner runs cleanup + -# publishes metrics, the others stay quiet. - -LEADER_KEY = 'cleanup_leader' # actual backend key: 'cleanup_leader' -LEASE_TTL = 30 # seconds — leader loses the lease after this without a renew -LEASE_RENEW = 10 # seconds — must be < LEASE_TTL/2 so two missed renews still beat the TTL _PROCESSOR_QUOTA_PREFIX = 'processor_quota::' @@ -104,13 +92,6 @@ def _sweep_processor_transform(existing: Any, *, now: float) -> dict[str, dict[s return _clean_processor_reservations(existing, now=now) -def _renew_if_owner(current: str | None, *, owner: str) -> str | None: - """``update_atomic`` transform: only re-write the lease if it is still mine.""" - if current == owner: - return owner - return None - - class ServerState: """Unified server state management class. @@ -123,9 +104,17 @@ class ServerState: - :class:`ConfigManager` — key-value configuration Each Ray Serve worker owns one process-local instance, bound directly to a - shared :class:`StateBackend`. The cleanup loop is started from the - deployment's FastAPI ``lifespan`` startup hook and only runs in the worker - that wins the cleanup-leader lease — see :meth:`_leader_loop`. + shared :class:`StateBackend`. + + Cleanup start-up: NOT from FastAPI lifespan startup. Ray Serve binds + ``servable_object`` *after* lifespan startup, so ``start_cleanup_task()`` is + lazy-started on the first request by + ``deployment.LazyCleanupMixin._ensure_state_cleanup_started`` (the reason is + spelled out in ``build_deployment_app``'s lifespan comment). Cleanup + orchestration and cleanup-leader election live in + :class:`ResourceCleanupCoordinator`; resource-count publishing lives in + :class:`ResourceCountPublisher`. ``start_cleanup_task`` is idempotent, which is + what makes per-request invocation safe. """ def __init__( @@ -148,20 +137,41 @@ def __init__( self.expiration_timeout = expiration_timeout self.cleanup_interval = cleanup_interval - self._cleanup_task: asyncio.Task | None = None - self._cleanup_running = False - - # Leader election + metrics-publish loop state. ``metrics_update_interval`` - # is a typed parameter (a misspelled key now fails loudly rather than - # being silently ignored); it controls how often the leader pushes counts - # into the MetricsRegistry cache. - self._leader_id = uuid.uuid4().hex - self._is_leader = False - self._leader_task: asyncio.Task | None = None - self._leader_running = False - self._metrics_publish_task: asyncio.Task | None = None - self._metrics_publish_running = False - self._metrics_update_interval: float = float(metrics_update_interval) + + # Cleanup orchestration, leader election and resource-count publishing are + # delegated. ``metrics_update_interval`` controls how often the leader pushes + # counts into the MetricsRegistry cache. ``on_become_leader`` / + # ``on_lose_leader`` wire leader identity to the publisher's lifecycle so only + # the leader publishes, while keeping the coordinator itself free + # of any telemetry dependency. + _managers = { + 'sessions': self._session_mgr, + 'models': self._model_mgr, + 'sampling_sessions': self._sampling_mgr, + 'futures': self._future_mgr, + } + self._count_publisher = ResourceCountPublisher( + [ + ('active_sessions', self._session_mgr), + ('active_models', self._model_mgr), + ('active_sampling_sessions', self._sampling_mgr), + ('active_futures', self._future_mgr), + ], + interval=float(metrics_update_interval), + ) + self._cleanup = ResourceCleanupCoordinator( + self._backend, + _managers, + cleanup_interval=cleanup_interval, + expiration_timeout=expiration_timeout, + sweep_processor_quotas=self.sweep_processor_quotas, + on_become_leader=self._count_publisher.start, + on_lose_leader=self._on_lose_leader, + ) + + async def _on_lose_leader(self) -> None: + await self._count_publisher.stop() + self._count_publisher.clear() async def get_capacity_info(self) -> dict[str, int]: return await self._model_mgr.get_capacity_info() @@ -337,8 +347,7 @@ async def reserve_processor_quota( ttl=self._processor_quota_ttl(lease_seconds), ) if not isinstance(reservations, dict) or processor_id not in reservations: - raise ResourceQuotaExceededError( - f'Per-user processor quota ({limit}) reached for token {token[:8]}...') + raise ResourceQuotaExceededError(f'Per-user processor quota ({limit}) reached for token {token[:8]}...') async def renew_processor_quota( self, @@ -528,227 +537,46 @@ async def count_config(self) -> int: async def cleanup_expired_resources(self) -> dict[str, int]: """Clean up expired sessions, models, sampling_sessions, and futures. - Sessions expire based on last_heartbeat (or created_at). Models and - sampling sessions are also cascade-expired when their owning session - expires. Futures expire based on updated_at (or created_at). - - Returns: - Dict with counts of cleaned up resources by type. + Delegates to :class:`ResourceCleanupCoordinator`. """ - current_time = time.time() - cutoff_time = current_time - self.expiration_timeout - - # Determine expired sessions and remove them in a SINGLE pass, then - # cascade the SAME set to dependent resources. Using one authoritative - # set (rather than a separate expiry scan followed by a second scan in - # cleanup) closes the TOCTOU window where a session touched mid-cleanup - # could survive removal while its children were cascade-deleted. - expired_session_ids, sessions_removed = await self._session_mgr.collect_and_remove_expired(cutoff_time) - - models_removed = await self._model_mgr.cleanup_expired(cutoff_time, expired_session_ids=expired_session_ids) - samplings_removed = await self._sampling_mgr.cleanup_expired( - cutoff_time, expired_session_ids=expired_session_ids) - - alive_replica_ids = await self._model_mgr.get_alive_replica_ids(self.expiration_timeout) - futures_removed = await self._future_mgr.cleanup_expired(cutoff_time, alive_replica_ids=alive_replica_ids) - await self.sweep_processor_quotas() - - return { - 'sessions': sessions_removed, - 'models': models_removed, - 'sampling_sessions': samplings_removed, - 'futures': futures_removed, - } + return await self._cleanup.cleanup_expired_resources() async def touch_replica_last_seen(self, replica_id: str) -> None: """Refresh a replica's liveness timestamp in the shared registry.""" await self._model_mgr.touch_replica_last_seen(replica_id) - async def _cleanup_loop(self) -> None: - """Background task that periodically cleans up expired resources. + # ----- Cleanup + leader election (delegated to ResourceCleanupCoordinator) ----- - Gated by leader election — non-leader workers skip the actual cleanup - so the same backend isn't swept 4× by 4 deployment workers. - """ - while self._cleanup_running: - try: - await asyncio.sleep(self.cleanup_interval) - if not self._is_leader: - continue - stats = await self.cleanup_expired_resources() - if any(stats.values()): - logger.debug(f'[ServerState Cleanup] Removed expired resources: {stats}') - except asyncio.CancelledError: - break - except Exception as e: - logger.warning(f'[ServerState Cleanup] Error during cleanup: {e}') - continue - - # ----- Leader election + metrics publish ----- - - async def _leader_loop(self) -> None: - """Acquire and renew the cleanup-leader lease every LEASE_RENEW seconds.""" - await self._try_acquire_or_renew() # Race for leadership at startup - while self._leader_running: - try: - await asyncio.sleep(LEASE_RENEW) - await self._try_acquire_or_renew() - except asyncio.CancelledError: - break - except Exception as e: - logger.warning(f'[ServerState Leader] renew error: {e}') - continue + @property + def _is_leader(self) -> bool: + return self._cleanup._is_leader - async def _try_acquire_or_renew(self) -> None: - was_leader = self._is_leader - try: - if self._is_leader: - val = await self._backend.update_atomic( - LEADER_KEY, - functools.partial(_renew_if_owner, owner=self._leader_id), - ttl=LEASE_TTL, - ) - self._is_leader = (val == self._leader_id) - else: - self._is_leader = await self._backend.set_nx(LEADER_KEY, self._leader_id, ttl=LEASE_TTL) - except Exception as e: - logger.warning(f'[ServerState Leader] backend error during election: {e}') - self._is_leader = False - if was_leader: - # Our renewal failed but our lease value may still be sitting in - # the backend, so a plain ``set_nx`` would keep returning False - # for up to LEASE_TTL and leadership would stall unclaimed. - # Best-effort delete ONLY when we were the leader (never steal a - # lease another replica legitimately holds), swallowing errors so - # a delete failure cannot escape the election loop. The next tick - # can then re-acquire immediately. - try: - await self._backend.delete(LEADER_KEY) - except Exception: - pass - - if self._is_leader and not was_leader: - await self._on_become_leader() - elif not self._is_leader and was_leader: - await self._on_lose_leader() - - async def _on_become_leader(self) -> None: - logger.info(f'[ServerState] became cleanup leader (id={self._leader_id[:8]})') - # Start pushing resource counts so the four ObservableGauges in the - # MetricsRegistry have a single source of truth across deployments. - if self._metrics_publish_task is None or self._metrics_publish_task.done(): - self._metrics_publish_running = True - self._metrics_publish_task = asyncio.create_task(self._metrics_publish_loop()) + @property + def _leader_id(self) -> str: + return self._cleanup._leader_id - async def _on_lose_leader(self) -> None: - logger.warning(f'[ServerState] lost cleanup leadership (id={self._leader_id[:8]})') - self._metrics_publish_running = False - if self._metrics_publish_task is not None: - self._metrics_publish_task.cancel() - try: - await self._metrics_publish_task - except (asyncio.CancelledError, Exception): - pass - self._metrics_publish_task = None - # Clear this worker's resource-gauge cache after cancelling the publish - # task. Across replicas the old leader's MetricsRegistry cache lives in - # a different process, and after handover its publish loop is cancelled - # and never overwrites the cache again — so without this zeroing the - # stale worker would keep emitting its last counts forever. The new - # leader publishes the authoritative counts from its own process. - MetricsRegistry.get().clear_resource_counts() - - async def _metrics_publish_loop(self) -> None: - """Push resource counts into the MetricsRegistry cache every N seconds. - - Only runs while this ``ServerState`` holds the cleanup-leader lease. - The ObservableGauges registered by :class:`MetricsRegistry` read the - cache at OTEL export time and report whatever was pushed last. - """ - registry = MetricsRegistry.get() - sources = ( - ('active_sessions', self._session_mgr), - ('active_models', self._model_mgr), - ('active_sampling_sessions', self._sampling_mgr), - ('active_futures', self._future_mgr), - ) - while self._metrics_publish_running: - try: - await asyncio.sleep(self._metrics_update_interval) - if not self._is_leader: - continue - for name, mgr in sources: - registry.set_resource_count(name, await mgr.count()) - except asyncio.CancelledError: - break - except Exception as e: - logger.debug(f'[ServerState] Error publishing metrics: {e}') - continue + async def _try_acquire_or_renew(self) -> None: + await self._cleanup._try_acquire_or_renew() async def start_cleanup_task(self) -> bool: """Start the background cleanup + leader-election tasks. - Returns: - True if tasks were started, False if already running. + Returns True if tasks were started, False if already running. The + idempotency guard lives inside the coordinator's ``start`` so a + per-request lazy invocation cannot double-start. """ - if self._cleanup_running: - return False - # Rebuild in-memory indexes from backend data - await self._rebuild_indexes() - self._cleanup_running = True - self._cleanup_task = asyncio.create_task(self._cleanup_loop()) - self._leader_running = True - self._leader_task = asyncio.create_task(self._leader_loop()) - return True - - async def _rebuild_indexes(self) -> None: - """Rebuild in-memory indexes from backend data after startup.""" - # Rebuild model indexes - await self._model_mgr.rebuild_indexes() + return await self._cleanup.start() async def stop_cleanup_task(self) -> bool: """Stop the background cleanup + leader-election tasks. - Returns: - True if tasks were stopped, False if not running. + Returns True if tasks were stopped, False if not running. """ - if not self._cleanup_running: - return False - self._cleanup_running = False - if self._cleanup_task: - self._cleanup_task.cancel() - self._cleanup_task = None - self._leader_running = False - if self._leader_task: - self._leader_task.cancel() - self._leader_task = None - if self._is_leader: - # Release callback registration; the lease itself expires on its own - # TTL — update_atomic can't express "atomic delete", so we accept a - # short outage where the gauge reads 0 between leaders. - await self._on_lose_leader() - self._is_leader = False - return True + return await self._cleanup.stop() async def get_cleanup_stats(self) -> dict[str, Any]: - """Get current cleanup configuration and resource counts. - - Returns: - Dict with cleanup configuration and task status. - """ - return { - 'expiration_timeout': self.expiration_timeout, - 'cleanup_interval': self.cleanup_interval, - 'cleanup_running': self._cleanup_running, - 'is_leader': self._is_leader, - 'leader_id': self._leader_id, - 'resource_counts': { - 'sessions': await self._session_mgr.count(), - 'models': await self._model_mgr.count(), - 'sampling_sessions': await self._sampling_mgr.count(), - 'futures': await self._future_mgr.count(), - }, - } + """Get current cleanup configuration and resource counts.""" + return await self._cleanup.get_cleanup_stats() # --------------------------------------------------------------------------- @@ -758,9 +586,9 @@ async def get_cleanup_stats(self) -> dict[str, Any]: # Each Ray Serve worker binds one ``ServerState`` instance to the shared # ``StateBackend`` for the lifetime of the process — the cleanup loop and # leader-election loop are started exactly once per worker (see -# ``start_cleanup_task``). Callers use ``actor_name`` as the cache key purely -# for per-process deduplication; cross-worker coordination happens inside the -# shared backend, not in this dict. +# ``start_cleanup_task``). Callers use ``cache_key`` purely for per-process +# deduplication; cross-worker coordination happens inside the shared backend, +# not in this dict. _PROCESS_STATE_CACHE: dict[str, ServerState] = {} @@ -772,7 +600,7 @@ async def get_cleanup_stats(self) -> dict[str, Any]: _DEFAULT_METRICS_UPDATE_INTERVAL = 15.0 -def get_server_state(actor_name: str = 'twinkle_server_state', +def get_server_state(cache_key: str = 'twinkle_server_state', backend: StateBackend | None = None, persistence_config: PersistenceConfig | None = None, expiration_timeout: float | None = None, @@ -781,14 +609,15 @@ def get_server_state(actor_name: str = 'twinkle_server_state', metrics_update_interval: float | None = None) -> ServerState: """Return a process-local :class:`ServerState` bound directly to the backend. - Within one process the same ``actor_name`` returns the same cached instance + Within one process the same ``cache_key`` returns the same cached instance so repeated callers share one ``ServerState`` and the cleanup loop is started exactly once. Cross-worker consistency comes from the shared :class:`StateBackend` rather than from any singleton in this process. Args: - actor_name: Cache key for the per-process ``ServerState`` instance. - The legacy parameter name is kept for call-site compatibility. + cache_key: Cache key for the per-process ``ServerState`` instance. + (Formerly ``actor_name``; the parameter never carried actor + semantics.) backend: Optional :class:`StateBackend` to inject. When ``None`` a backend is built from ``persistence_config`` (or env vars) via :func:`create_backend`. @@ -805,7 +634,7 @@ def get_server_state(actor_name: str = 'twinkle_server_state', if backend is None and persistence_config is None: persistence_config = PersistenceConfig.from_env() - cached = _PROCESS_STATE_CACHE.get(actor_name) + cached = _PROCESS_STATE_CACHE.get(cache_key) if cached is not None: return cached @@ -839,7 +668,7 @@ def _resolve(explicit, env_value, default): per_token_model_limit=per_token_model_limit, metrics_update_interval=metrics_update_interval, ) - _PROCESS_STATE_CACHE[actor_name] = state + _PROCESS_STATE_CACHE[cache_key] = state logger.info( 'ServerState policy in effect: per_token_model_limit=%s expiration_timeout=%s ' 'cleanup_interval=%s metrics_update_interval=%s (resolution: explicit>env>default)', per_token_model_limit, diff --git a/src/twinkle/server/utils/task_errors.py b/src/twinkle/server/task_errors.py similarity index 100% rename from src/twinkle/server/utils/task_errors.py rename to src/twinkle/server/task_errors.py diff --git a/src/twinkle/server/utils/task_queue/__init__.py b/src/twinkle/server/task_queue/__init__.py similarity index 91% rename from src/twinkle/server/utils/task_queue/__init__.py rename to src/twinkle/server/task_queue/__init__.py index d77dfe18..bf539161 100644 --- a/src/twinkle/server/utils/task_queue/__init__.py +++ b/src/twinkle/server/task_queue/__init__.py @@ -2,7 +2,7 @@ """ Task Queue package. -Public exports (backward-compatible with the former task_queue.py module): +Public exports: - TaskStatus - task lifecycle enum - QueueState - queue state enum for tinker client compatibility - TaskQueueConfig - queue and rate-limit configuration dataclass diff --git a/src/twinkle/server/task_queue/backend_gate.py b/src/twinkle/server/task_queue/backend_gate.py new file mode 100644 index 00000000..82d17c1e --- /dev/null +++ b/src/twinkle/server/task_queue/backend_gate.py @@ -0,0 +1,121 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""The blocking-backend-call boundary and the per-replica admission gate. + +Conceptually unrelated to "task queue": this is where an event-loop coroutine hands +work to a thread and waits. It was welded onto ``TaskQueueMixin`` because +``tests/server/static/test_no_direct_backend_call.py`` forces every backend call +through ``call_backend``, which made that mixin the sole path to the backend -- one +static check binding two concepts. + +This class is a pure callable wrapper: it receives an already-bound ``fn`` from the +caller and only does ``executor.submit(functools.partial(fn, ...))``. It never writes a +``self.model.`` attribute chain and never holds a backend reference, so it is +invisible to the AST scan in ``test_no_direct_backend_call.py`` and needs no exemption +entry. +""" +from __future__ import annotations + +import asyncio +import contextlib +import functools +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from typing import Any + +from .types import BackendBusyError + + +class BackendGate: + """Owns the two executors, the admission lock, and the poison event. + + Construction must happen inside a running event loop: ``asyncio.Lock()`` and + ``asyncio.Event()`` bind to the loop that creates them. This holds today because + ``_init_task_queue`` is called from Ray Serve's ``async def __init__``. + """ + + def __init__(self, *, enable_admission_gate: bool = False) -> None: + self._executor = ThreadPoolExecutor(thread_name_prefix='twinkle-backend') + self._probe_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='twinkle-backend-probe') + self._admission: asyncio.Lock | None = asyncio.Lock() if enable_admission_gate else None + self._poisoned = asyncio.Event() + + async def _acquire(self, gate: asyncio.Lock) -> None: + if self._poisoned.is_set(): + raise BackendBusyError('This replica is waiting for a timed-out backend call to exit.') + if not gate.locked(): + await gate.acquire() + else: + acquire_task = asyncio.create_task(gate.acquire()) + poison_task = asyncio.create_task(self._poisoned.wait()) + try: + done, _ = await asyncio.wait((acquire_task, poison_task), return_when=asyncio.FIRST_COMPLETED) + except asyncio.CancelledError: + acquire_task.cancel() + poison_task.cancel() + await asyncio.gather(acquire_task, poison_task, return_exceptions=True) + if acquire_task.done() and not acquire_task.cancelled() and acquire_task.result(): + gate.release() + raise + if poison_task in done and self._poisoned.is_set(): + if acquire_task.done() and not acquire_task.cancelled() and acquire_task.result(): + gate.release() + else: + acquire_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await acquire_task + raise BackendBusyError('This replica is waiting for a timed-out backend call to exit.') + poison_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await poison_task + await acquire_task + if self._poisoned.is_set(): + gate.release() + raise BackendBusyError('This replica is waiting for a timed-out backend call to exit.') + + async def call(self, fn: Callable[..., Any], /, *args: Any, admit: bool = True, **kwargs: Any) -> Any: + """Run one backend call outside the event loop. + + Normal model calls serialize through the admission gate. If the awaiting task + times out while its thread is still running, the gate is poisoned: waiters fail + immediately until that thread exits. Health probes bypass the gate and use a + reserved executor thread. Sampler deployments disable the gate because their + backend owns request concurrency. + """ + loop = asyncio.get_running_loop() + gate = self._admission if admit else None + if gate is not None: + await self._acquire(gate) + + executor = self._executor if admit else self._probe_executor + try: + concurrent_future = executor.submit(functools.partial(fn, *args, **kwargs)) + except Exception: + if gate is not None and gate.locked(): + gate.release() + raise + + if gate is not None: + + def release_gate(_future) -> None: + + def release() -> None: + self._poisoned.clear() + if gate.locked(): + gate.release() + + with contextlib.suppress(RuntimeError): + loop.call_soon_threadsafe(release) + + concurrent_future.add_done_callback(release_gate) + + try: + return await asyncio.wrap_future(concurrent_future, loop=loop) + except asyncio.CancelledError: + if gate is not None and concurrent_future.running(): + self._poisoned.set() + raise + + def shutdown(self) -> None: + # Do not wait on threads that may be leaked on a timed-out backend call. + self._executor.shutdown(wait=False, cancel_futures=True) + self._probe_executor.shutdown(wait=False, cancel_futures=True) diff --git a/src/twinkle/server/utils/task_queue/config.py b/src/twinkle/server/task_queue/config.py similarity index 100% rename from src/twinkle/server/utils/task_queue/config.py rename to src/twinkle/server/task_queue/config.py diff --git a/src/twinkle/server/utils/task_queue/mixin.py b/src/twinkle/server/task_queue/mixin.py similarity index 80% rename from src/twinkle/server/utils/task_queue/mixin.py rename to src/twinkle/server/task_queue/mixin.py index fea48fbb..0d89d4ff 100644 --- a/src/twinkle/server/utils/task_queue/mixin.py +++ b/src/twinkle/server/task_queue/mixin.py @@ -8,25 +8,23 @@ from __future__ import annotations import asyncio -import contextlib -import functools import time import traceback import uuid from collections.abc import Callable, Coroutine -from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Any from twinkle.server.exceptions import BatchSizeError, ConfigError, InputTokensExceededError, RateLimitExceededError from twinkle.server.lifecycle.envelope import envelope_from_record from twinkle.server.lifecycle.poll_config import long_poll_window from twinkle.server.state.models import FutureFailureRecord -from twinkle.server.telemetry.middleware import get_task_metrics +from twinkle.server.telemetry.metrics import get_task_metrics from twinkle.utils.logger import get_logger from twinkle_client.types.lifecycle import TERMINAL_STATUSES, TaskEnvelope +from .backend_gate import BackendGate from .config import TaskQueueConfig from .rate_limiter import RateLimiter -from .types import BackendBusyError, QueuedTask, QueueState, TaskStatus +from .types import QueuedTask, QueueState, TaskStatus from .worker import ComputeWorker if TYPE_CHECKING: @@ -121,87 +119,28 @@ def _init_task_queue( on_backend_timeout=on_backend_timeout, ) - self._backend_executor = ThreadPoolExecutor(thread_name_prefix='twinkle-backend') - self._backend_probe_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='twinkle-backend-probe') - self._backend_admission: asyncio.Lock | None = asyncio.Lock() if enable_admission_gate else None - self._backend_poisoned = asyncio.Event() + self._backend_gate = BackendGate(enable_admission_gate=enable_admission_gate) self._event_loop: asyncio.AbstractEventLoop | None = None - async def _acquire_backend_gate(self, gate: asyncio.Lock) -> None: - if self._backend_poisoned.is_set(): - raise BackendBusyError('This replica is waiting for a timed-out backend call to exit.') - if not gate.locked(): - await gate.acquire() - else: - acquire_task = asyncio.create_task(gate.acquire()) - poison_task = asyncio.create_task(self._backend_poisoned.wait()) - try: - done, _ = await asyncio.wait((acquire_task, poison_task), return_when=asyncio.FIRST_COMPLETED) - except asyncio.CancelledError: - acquire_task.cancel() - poison_task.cancel() - await asyncio.gather(acquire_task, poison_task, return_exceptions=True) - if acquire_task.done() and not acquire_task.cancelled() and acquire_task.result(): - gate.release() - raise - if poison_task in done and self._backend_poisoned.is_set(): - if acquire_task.done() and not acquire_task.cancelled() and acquire_task.result(): - gate.release() - else: - acquire_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await acquire_task - raise BackendBusyError('This replica is waiting for a timed-out backend call to exit.') - poison_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await poison_task - await acquire_task - if self._backend_poisoned.is_set(): - gate.release() - raise BackendBusyError('This replica is waiting for a timed-out backend call to exit.') + @property + def task_queue_config(self) -> TaskQueueConfig: + """The deployment's validated queue config. + + Public because four modules outside this file read it; it was + ``_task_queue_config`` (name-private, interface-public), which meant the + Host_Protocol could not declare it without declaring a private name. + """ + return self._task_queue_config async def call_backend(self, fn: Callable[..., Any], /, *args: Any, admit: bool = True, **kwargs: Any) -> Any: """Run one backend call outside the event loop. - Normal model calls serialize through the admission gate. If the awaiting - task times out while its thread is still running, the gate is poisoned: - waiters fail immediately until that thread exits. Health probes bypass the - gate and use a reserved executor thread. Sampler deployments disable the - gate because their backend owns request concurrency. + Single delegation to :class:`BackendGate`. The name and call shape are + load-bearing: ``test_no_direct_backend_call.py`` keys on ``self.call_backend`` + call sites, so keeping it here means the check's predicate and its exemption + table need no change. """ - loop = asyncio.get_running_loop() - gate = self._backend_admission if admit else None - if gate is not None: - await self._acquire_backend_gate(gate) - - executor = self._backend_executor if admit else self._backend_probe_executor - try: - concurrent_future = executor.submit(functools.partial(fn, *args, **kwargs)) - except Exception: - if gate is not None and gate.locked(): - gate.release() - raise - - if gate is not None: - - def release_gate(_future) -> None: - - def release() -> None: - self._backend_poisoned.clear() - if gate.locked(): - gate.release() - - with contextlib.suppress(RuntimeError): - loop.call_soon_threadsafe(release) - - concurrent_future.add_done_callback(release_gate) - - try: - return await asyncio.wrap_future(concurrent_future, loop=loop) - except asyncio.CancelledError: - if gate is not None and concurrent_future.running(): - self._backend_poisoned.set() - raise + return await self._backend_gate.call(fn, *args, admit=admit, **kwargs) def _future_deadline(self) -> float: ttl = getattr(self, '_future_absolute_ttl', self._task_queue_config.absolute_future_ttl(1)) @@ -229,7 +168,7 @@ async def _perform_preflight_checks( Returns ``None`` when every check passes. On failure it RAISES a ``RequestRejectedError`` subclass -- the Decision_Boundary is this line, and raising before any ``store_future_status`` call is what guarantees zero - future writes for a rejected request (Property 3). It writes no FAILED + future writes for a rejected request. It writes no FAILED record and returns no ``_error`` marker. """ if not token or not self._task_queue_config.enabled: @@ -318,11 +257,7 @@ async def _schedule_task( f'model_id={model_id}, queue_key={queue_key}, ' f'queue_depth={q.qsize()}, input_tokens={input_tokens}') - self._compute_worker.new_task_event.set() - - if self._task_metrics: - total_depth = self._compute_worker.total_queued() - self._task_metrics.queue_depth.set(total_depth, tags={'deployment': self._deployment_name}) + self._compute_worker.notify_new_task() return {'request_id': request_id, 'model_id': model_id} @@ -513,9 +448,5 @@ async def shutdown_task_queue(self) -> None: """Gracefully shut down the compute queue and release resources.""" await self._rate_limiter.stop_cleanup_task() await self._compute_worker.stop() - # Do not wait on threads that may be leaked on a timed-out backend call. - if getattr(self, '_backend_executor', None) is not None: - self._backend_executor.shutdown(wait=False, cancel_futures=True) - if getattr(self, '_backend_probe_executor', None) is not None: - self._backend_probe_executor.shutdown(wait=False, cancel_futures=True) + self._backend_gate.shutdown() logger.debug('[TaskQueue] Task queue shutdown complete') diff --git a/src/twinkle/server/utils/task_queue/rate_limiter.py b/src/twinkle/server/task_queue/rate_limiter.py similarity index 99% rename from src/twinkle/server/utils/task_queue/rate_limiter.py rename to src/twinkle/server/task_queue/rate_limiter.py index 8f1dbe97..7e301674 100644 --- a/src/twinkle/server/utils/task_queue/rate_limiter.py +++ b/src/twinkle/server/task_queue/rate_limiter.py @@ -55,7 +55,7 @@ def __init__( will be removed. Default is 10.0 (10x the window). token_cleanup_interval: How often to run the cleanup task in seconds. Default is 60.0 (every minute). - active_tokens_gauge: Optional gauge adapter (see twinkle.server.telemetry.middleware) + active_tokens_gauge: Optional gauge adapter (see twinkle.server.telemetry.metrics) for tracking the active token count. deployment_name: Deployment name for metrics labels. """ diff --git a/src/twinkle/server/utils/task_queue/types.py b/src/twinkle/server/task_queue/types.py similarity index 100% rename from src/twinkle/server/utils/task_queue/types.py rename to src/twinkle/server/task_queue/types.py diff --git a/src/twinkle/server/utils/task_queue/worker.py b/src/twinkle/server/task_queue/worker.py similarity index 95% rename from src/twinkle/server/utils/task_queue/worker.py rename to src/twinkle/server/task_queue/worker.py index f163a966..aba5145c 100644 --- a/src/twinkle/server/utils/task_queue/worker.py +++ b/src/twinkle/server/task_queue/worker.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: from twinkle.server.state import ServerState - from twinkle.server.telemetry.middleware import TaskMetrics + from twinkle.server.telemetry.metrics import TaskMetrics logger = get_logger() @@ -141,6 +141,16 @@ def total_queued(self) -> int: """Total number of pending tasks across all per-key queues.""" return sum(q.qsize() for q in self.task_queues.values()) + def notify_new_task(self) -> None: + """Wake the worker loop and refresh the queue-depth gauge. + + Producers (the mixin) call this instead of reaching into the worker's + ``new_task_event`` directly; it is also the single write point of the + queue-depth gauge, so the gauge has one writer rather than two. + """ + self.new_task_event.set() + self._record_queue_depth() + # ------------------------------------------------------------------ # Metrics helpers # ------------------------------------------------------------------ @@ -163,6 +173,12 @@ def _record_execution_time(self, task_type: str, exec_time: float) -> None: 'task_type': task_type, }) + def _record_queue_depth(self) -> None: + """Single writer of the queue-depth gauge.""" + if self._task_metrics: + total_depth = sum(qq.qsize() for qq in self.task_queues.values()) + self._task_metrics.queue_depth.set(total_depth, tags={'deployment': self._deployment_name}) + def _record_queue_metrics(self, task_type: str, queue_wait: float) -> None: """Observe queue wait time and update current queue depth if metrics are enabled.""" if self._task_metrics: @@ -171,8 +187,7 @@ def _record_queue_metrics(self, task_type: str, queue_wait: float) -> None: 'deployment': self._deployment_name, 'task_type': task_type, }) - total_depth = sum(qq.qsize() for qq in self.task_queues.values()) - self._task_metrics.queue_depth.set(total_depth, tags={'deployment': self._deployment_name}) + self._record_queue_depth() # ------------------------------------------------------------------ @@ -300,8 +315,7 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue logger.error(f'[ComputeWorker] Task {task.request_id} TIMEOUT after {exec_time:.2f}s, ' f'type={task_type}, queue_key={queue_key}') # asyncio.TimeoutError and Ray_Get_Timeout are 504/Server. - await self._store_task_failed( - task, error, QueueState.ACTIVE.value, reason_code='execution_timeout') + await self._store_task_failed(task, error, QueueState.ACTIVE.value, reason_code='execution_timeout') # Probe actor liveness after a timeout so an operator learns the replica's # state without waiting for a second request to also time out. if self._on_backend_timeout is not None: @@ -326,8 +340,7 @@ async def _execute_task(self, task: QueuedTask, queue_key: str, q: asyncio.Queue logger.error(f'[ComputeWorker] Task {task.request_id} REFUSED (admission gate held) after ' f'{exec_time:.2f}s, type={task_type}, queue_key={queue_key}') # Gate held by a leaked timed-out call -> 503/Server. - await self._store_task_failed( - task, error, QueueState.ACTIVE.value, reason_code='backend_gate_unavailable') + await self._store_task_failed(task, error, QueueState.ACTIVE.value, reason_code='backend_gate_unavailable') except TwinkleServerError as exc: # A typed server error carries its own status + category (e.g. # ResourceNotFoundError = 404/User from a deferred diff --git a/src/twinkle/server/telemetry/__init__.py b/src/twinkle/server/telemetry/__init__.py index 21803431..ded3157e 100644 --- a/src/twinkle/server/telemetry/__init__.py +++ b/src/twinkle/server/telemetry/__init__.py @@ -1,4 +1,3 @@ -from twinkle.server.config.telemetry import TelemetryConfig from .metrics import MetricsRegistry from .provider import get_meter, init_telemetry, shutdown_telemetry from .tracing import extract_context, get_current_span, get_tracer, inject_context @@ -6,7 +5,6 @@ __all__ = [ 'MetricsRegistry', - 'TelemetryConfig', 'get_meter', 'init_telemetry', 'shutdown_telemetry', diff --git a/src/twinkle/server/telemetry/http_middleware.py b/src/twinkle/server/telemetry/http_middleware.py new file mode 100644 index 00000000..beeee4e5 --- /dev/null +++ b/src/twinkle/server/telemetry/http_middleware.py @@ -0,0 +1,56 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""FastAPI HTTP request-metrics middleware. + +Split out of the former ``middleware.py``: the metric adapters and +containers live in ``metrics.py`` next to the ``MetricsRegistry``; this file holds only +the HTTP middleware factory, so a file named for HTTP middleware contains HTTP middleware +and the queue code no longer imports a module called ``middleware`` just to reach +``get_task_metrics``. +""" +from __future__ import annotations + +import time +from collections.abc import Callable +from typing import Any + +from twinkle.server.telemetry.metrics import MetricsRegistry + + +def create_metrics_middleware(deployment: str) -> Callable: + """Return a FastAPI ``http`` middleware that records request metrics. + + Usage inside a ``build_*_app()`` function:: + + from twinkle.server.telemetry.http_middleware import create_metrics_middleware + from twinkle.server.telemetry.tracing import create_tracing_middleware + + app.middleware('http')(verify_token) + app.middleware('http')(create_tracing_middleware("Model")) + app.middleware('http')(create_metrics_middleware("Model")) # outermost + + FastAPI executes middleware in LIFO order, so the **last** middleware + registered is the outermost wrapper. Register metrics last so its + latency observation covers the full request path including tracing + overhead and authentication. + """ + + async def metrics_middleware(request: Any, call_next: Callable) -> Any: + start = time.monotonic() + response = await call_next(request) + elapsed = time.monotonic() - start + status = str(response.status_code) + method = request.scope['route'].path if 'route' in request.scope else request.url.path + m = MetricsRegistry.get().request_metrics(deployment) + m.requests_total.inc(tags={ + 'deployment': deployment, + 'method': method, + 'status': status, + }) + m.request_duration_seconds.observe( + elapsed, tags={ + 'deployment': deployment, + 'method': method, + }) + return response + + return metrics_middleware diff --git a/src/twinkle/server/telemetry/metrics.py b/src/twinkle/server/telemetry/metrics.py index 20518d36..58627770 100644 --- a/src/twinkle/server/telemetry/metrics.py +++ b/src/twinkle/server/telemetry/metrics.py @@ -1,7 +1,25 @@ -"""Twinkle Server metrics registry — low-invasiveness facade over OpenTelemetry metrics.""" +"""Twinkle Server metrics registry — low-invasiveness facade over OpenTelemetry metrics. + +Besides the :class:`MetricsRegistry` that declares the raw OTEL instruments, this module +holds the legacy-API adapter classes (``_Counter`` / ``_Histogram`` / ``_Gauge``), the +structured containers (:class:`TaskMetrics` / ``_RequestMetrics``) and ``get_task_metrics``. +Keeping the metric types and the registry in one file (with the HTTP middleware factory in +``http_middleware.py``) avoids a ``metrics <-> adapters`` import cycle. + +Per-deployment adapters are cached on the *registry instance* +(``_task_metrics`` / ``_request_metrics``), not at module level. That is load-bearing: the +adapters hold bound instrument objects, and ``reset()`` swaps the singleton precisely in +order to rebind them to a real MeterProvider (``worker_init.ensure_telemetry_initialized`` +does ``init_telemetry()`` -> ``reset()``). A module-level cache survived that swap, so +anything that called ``get_task_metrics`` before ``init_telemetry`` cached NoOp instruments +for the life of the process. +""" from __future__ import annotations +from pydantic import BaseModel, ConfigDict +from typing import Any + from .provider import get_meter try: @@ -16,6 +34,91 @@ ('active_futures', 'twinkle.futures.active', 'Number of pending futures/tasks'), ) +# --------------------------------------------------------------------------- +# Adapter classes – wrap OTEL instruments to expose the legacy Ray-style API +# (``.inc(tags=...)`` / ``.set(value, tags=...)`` / ``.observe(value, tags=...)``) +# while delegating all measurements to OpenTelemetry. +# --------------------------------------------------------------------------- + + +class _Counter: + """Adapter mapping ``.inc(value, tags=...)`` to ``otel_counter.add()``.""" + + def __init__(self, instrument: Any) -> None: + self._instrument = instrument + + def inc(self, value: float = 1.0, tags: dict[str, str] | None = None) -> None: + self._instrument.add(value, attributes=tags or {}) + + +class _Histogram: + """Adapter mapping ``.observe(value, tags=...)`` to ``otel_histogram.record()``.""" + + def __init__(self, instrument: Any) -> None: + self._instrument = instrument + + def observe(self, value: float, tags: dict[str, str] | None = None) -> None: + self._instrument.record(value, attributes=tags or {}) + + +class _Gauge: + """Adapter mapping ``.set(value, tags=...)`` onto an OTEL UpDownCounter. + + OpenTelemetry up/down counters take *deltas*, not absolute values, so we + track the last reported value per attribute combination and emit the + incremental change. State is held per adapter instance (= per deployment), + keyed by the frozen attribute tuple. + """ + + def __init__(self, instrument: Any) -> None: + self._instrument = instrument + self._last: dict[tuple, float] = {} + + def set(self, value: float, tags: dict[str, str] | None = None) -> None: + attrs = tags or {} + key = tuple(sorted(attrs.items())) + last = self._last.get(key, 0.0) + delta = value - last + if delta != 0: + self._instrument.add(delta, attributes=attrs) + self._last[key] = value + + +# --------------------------------------------------------------------------- +# Pydantic containers for structured metric access +# --------------------------------------------------------------------------- + + +class TaskMetrics(BaseModel): + """Task queue metrics container. + + Attributes: + queue_depth: Current number of queued tasks (gauge). + tasks_total: Total task completions (counter). + execution_seconds: Pure task execution time in seconds (histogram). + queue_wait_seconds: Time from enqueue to execution start (histogram). + rate_limit_rejections: Total rate-limit rejections (counter). + rate_limiter_active_tokens: Tokens tracked by rate limiter (gauge). + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + queue_depth: _Gauge + tasks_total: _Counter + execution_seconds: _Histogram + queue_wait_seconds: _Histogram + rate_limit_rejections: _Counter + rate_limiter_active_tokens: _Gauge + + +class _RequestMetrics(BaseModel): + """HTTP request metrics container (internal).""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + requests_total: _Counter + request_duration_seconds: _Histogram + class MetricsRegistry: """Centrally declares all metrics. Business code retrieves singleton via MetricsRegistry.get(). @@ -85,6 +188,11 @@ def __init__(self) -> None: description=description, ) + # Per-deployment adapter caches held on the instance so ``reset()`` (which + # swaps this singleton to rebind instruments) invalidates them. + self._task_metrics: dict[str, TaskMetrics] = {} + self._request_metrics: dict[str, _RequestMetrics] = {} + def _make_gauge_callback(self, name: str): """Build the sync OTEL callback that reads ``_resource_cache[name]``.""" @@ -93,6 +201,32 @@ def _callback(options): # noqa: ARG001 -- OTEL signature return _callback + # ----- Per-deployment adapter accessors ----- + + def task_metrics(self, deployment: str) -> TaskMetrics: + """Return (or build) the per-deployment task-queue metric adapters.""" + cached = self._task_metrics.get(deployment) + if cached is None: + cached = self._task_metrics[deployment] = TaskMetrics( + queue_depth=_Gauge(self.queue_depth), + tasks_total=_Counter(self.tasks_total), + execution_seconds=_Histogram(self.task_execution_seconds), + queue_wait_seconds=_Histogram(self.task_wait_seconds), + rate_limit_rejections=_Counter(self.rate_limit_rejections), + rate_limiter_active_tokens=_Gauge(self.rate_limiter_active_tokens), + ) + return cached + + def request_metrics(self, deployment: str) -> _RequestMetrics: + """Return (or build) the per-deployment HTTP request metric adapters.""" + cached = self._request_metrics.get(deployment) + if cached is None: + cached = self._request_metrics[deployment] = _RequestMetrics( + requests_total=_Counter(self.requests_total), + request_duration_seconds=_Histogram(self.request_duration_seconds), + ) + return cached + # ----- Push API for the cleanup leader ----- def set_resource_count(self, name: str, value: int) -> None: @@ -120,3 +254,17 @@ def get(cls) -> MetricsRegistry: def reset(cls) -> None: """Reset singleton (for testing or telemetry re-initialization).""" cls._instance = None + + +def get_task_metrics(deployment: str) -> TaskMetrics: + """Return the per-deployment task-queue metric adapters. + + Signature unchanged (``_init_task_queue`` needs no edit); the adapters are now + cached on the ``MetricsRegistry`` instance so ``reset()`` invalidates them. + """ + return MetricsRegistry.get().task_metrics(deployment) + + +def get_request_metrics(deployment: str) -> _RequestMetrics: + """Return the per-deployment HTTP request metric adapters.""" + return MetricsRegistry.get().request_metrics(deployment) diff --git a/src/twinkle/server/telemetry/middleware.py b/src/twinkle/server/telemetry/middleware.py deleted file mode 100644 index 4745dd9d..00000000 --- a/src/twinkle/server/telemetry/middleware.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright (c) ModelScope Contributors. All rights reserved. -""" -Central metrics module for Twinkle server observability. - -This module is a **back-compat keyword shim plus a real ``_Gauge`` adapter** over -the OpenTelemetry instruments declared in -:class:`twinkle.server.telemetry.metrics.MetricsRegistry`. ``_Counter`` and -``_Histogram`` are thin pass-throughs whose only role is to accept the legacy -Ray-style ``tags=`` keyword and forward it as OTEL's ``attributes=``; ``_Gauge`` -does real work — it translates the legacy ``set(value)`` API onto OTEL's -delta-based UpDownCounter by tracking the last reported value per attribute set. -Routing every measurement through OTEL while preserving the legacy call API -means existing call sites do not need to change. - -Public entry-points (unchanged signatures): - -* ``create_metrics_middleware(deployment)`` – FastAPI HTTP middleware -* ``get_task_metrics(deployment)`` – task-queue / rate-limit gauges -""" -from __future__ import annotations - -import time -from collections.abc import Callable -from pydantic import BaseModel, ConfigDict -from typing import Any - -from twinkle.server.telemetry.metrics import MetricsRegistry -from twinkle.utils.logger import get_logger - -logger = get_logger() - -# Per-process caches; each Ray Serve worker holds its own instance. -_task_metrics_cache: dict[str, TaskMetrics] = {} -_request_metrics_cache: dict[str, _RequestMetrics] = {} - -# --------------------------------------------------------------------------- -# Adapter classes – wrap OTEL instruments to expose the legacy Ray-style API -# (``.inc(tags=...)`` / ``.set(value, tags=...)`` / ``.observe(value, tags=...)``) -# while delegating all measurements to OpenTelemetry. -# --------------------------------------------------------------------------- - - -class _Counter: - """Adapter mapping ``.inc(value, tags=...)`` to ``otel_counter.add()``.""" - - def __init__(self, instrument: Any) -> None: - self._instrument = instrument - - def inc(self, value: float = 1.0, tags: dict[str, str] | None = None) -> None: - self._instrument.add(value, attributes=tags or {}) - - -class _Histogram: - """Adapter mapping ``.observe(value, tags=...)`` to ``otel_histogram.record()``.""" - - def __init__(self, instrument: Any) -> None: - self._instrument = instrument - - def observe(self, value: float, tags: dict[str, str] | None = None) -> None: - self._instrument.record(value, attributes=tags or {}) - - -class _Gauge: - """Adapter mapping ``.set(value, tags=...)`` onto an OTEL UpDownCounter. - - OpenTelemetry up/down counters take *deltas*, not absolute values, so we - track the last reported value per attribute combination and emit the - incremental change. State is held per adapter instance (= per deployment), - keyed by the frozen attribute tuple. - """ - - def __init__(self, instrument: Any) -> None: - self._instrument = instrument - self._last: dict[tuple, float] = {} - - def set(self, value: float, tags: dict[str, str] | None = None) -> None: - attrs = tags or {} - key = tuple(sorted(attrs.items())) - last = self._last.get(key, 0.0) - delta = value - last - if delta != 0: - self._instrument.add(delta, attributes=attrs) - self._last[key] = value - - -# --------------------------------------------------------------------------- -# Pydantic containers for structured metric access -# --------------------------------------------------------------------------- - - -class TaskMetrics(BaseModel): - """Task queue metrics container. - - Attributes: - queue_depth: Current number of queued tasks (gauge). - tasks_total: Total task completions (counter). - execution_seconds: Pure task execution time in seconds (histogram). - queue_wait_seconds: Time from enqueue to execution start (histogram). - rate_limit_rejections: Total rate-limit rejections (counter). - rate_limiter_active_tokens: Tokens tracked by rate limiter (gauge). - """ - - model_config = ConfigDict(arbitrary_types_allowed=True) - - queue_depth: _Gauge - tasks_total: _Counter - execution_seconds: _Histogram - queue_wait_seconds: _Histogram - rate_limit_rejections: _Counter - rate_limiter_active_tokens: _Gauge - - -class _RequestMetrics(BaseModel): - """HTTP request metrics container (internal).""" - - model_config = ConfigDict(arbitrary_types_allowed=True) - - requests_total: _Counter - request_duration_seconds: _Histogram - - -# --------------------------------------------------------------------------- -# A. Request-level metrics (FastAPI middleware) -# --------------------------------------------------------------------------- - - -def _get_request_metrics(deployment: str) -> _RequestMetrics: - """Return (or create) per-deployment HTTP request metric adapters.""" - if deployment in _request_metrics_cache: - return _request_metrics_cache[deployment] - - reg = MetricsRegistry.get() - metrics = _RequestMetrics( - requests_total=_Counter(reg.requests_total), - request_duration_seconds=_Histogram(reg.request_duration_seconds), - ) - _request_metrics_cache[deployment] = metrics - return metrics - - -def create_metrics_middleware(deployment: str) -> Callable: - """Return a FastAPI ``http`` middleware that records request metrics. - - Usage inside a ``build_*_app()`` function:: - - from twinkle.server.telemetry.middleware import create_metrics_middleware - from twinkle.server.telemetry.tracing import create_tracing_middleware - - app.middleware('http')(verify_token) - app.middleware('http')(create_tracing_middleware("Model")) - app.middleware('http')(create_metrics_middleware("Model")) # outermost - - FastAPI executes middleware in LIFO order, so the **last** middleware - registered is the outermost wrapper. Register metrics last so its - latency observation covers the full request path including tracing - overhead and authentication. - """ - - async def metrics_middleware(request: Any, call_next: Callable) -> Any: - start = time.monotonic() - response = await call_next(request) - elapsed = time.monotonic() - start - status = str(response.status_code) - method = request.scope['route'].path if 'route' in request.scope else request.url.path - m = _get_request_metrics(deployment) - m.requests_total.inc(tags={ - 'deployment': deployment, - 'method': method, - 'status': status, - }) - m.request_duration_seconds.observe( - elapsed, tags={ - 'deployment': deployment, - 'method': method, - }) - return response - - return metrics_middleware - - -# --------------------------------------------------------------------------- -# B. Task-queue metrics -# --------------------------------------------------------------------------- - - -def get_task_metrics(deployment: str) -> TaskMetrics: - """Return (or create) per-deployment task-queue metric adapters. - - Returns a :class:`TaskMetrics` container of adapter objects; the - adapters delegate every measurement to the OTEL instruments held by - :class:`twinkle.server.telemetry.metrics.MetricsRegistry`. A separate - adapter instance is cached per deployment so that gauge-state tracking - (last value per attribute set) stays isolated. - """ - if deployment in _task_metrics_cache: - return _task_metrics_cache[deployment] - - reg = MetricsRegistry.get() - metrics = TaskMetrics( - queue_depth=_Gauge(reg.queue_depth), - tasks_total=_Counter(reg.tasks_total), - execution_seconds=_Histogram(reg.task_execution_seconds), - queue_wait_seconds=_Histogram(reg.task_wait_seconds), - rate_limit_rejections=_Counter(reg.rate_limit_rejections), - rate_limiter_active_tokens=_Gauge(reg.rate_limiter_active_tokens), - ) - _task_metrics_cache[deployment] = metrics - return metrics diff --git a/src/twinkle/server/utils/__init__.py b/src/twinkle/server/utils/__init__.py index bddeed29..ab27cf29 100644 --- a/src/twinkle/server/utils/__init__.py +++ b/src/twinkle/server/utils/__init__.py @@ -1,5 +1,12 @@ # Copyright (c) ModelScope Contributors. All rights reserved. +"""Stateless server-side helpers. + +Deliberately re-exports only ``device_utils`` and ``template_utils``: the five call sites +that import through this bucket all want just those two (``get_template_for_model``, a +52-line string map, and ``wrap_builder_with_device_group_env``), while the code that +actually needs the queue / session-resource machinery imports the full path. Re-exporting +the mixins pulled the whole OpenTelemetry SDK into every one of those five importers +(``task_queue.mixin`` -> ``telemetry`` -> the OpenTelemetry SDK + OTLP exporter). +""" from .device_utils import auto_fill_device_group_visible_devices, wrap_builder_with_device_group_env -from .session_resource import AdapterManagerMixin, ProcessorManagerMixin, SessionResourceMixin -from .task_queue import QueueState, RateLimiter, TaskQueueConfig, TaskQueueMixin, TaskStatus from .template_utils import get_template_for_model diff --git a/src/twinkle/server/validation/__init__.py b/src/twinkle/server/validation/__init__.py index 6ff764dd..36335a7a 100644 --- a/src/twinkle/server/validation/__init__.py +++ b/src/twinkle/server/validation/__init__.py @@ -16,16 +16,11 @@ Both read *declared* metadata, so neither can reject a valid request. A third, heuristic check over passthrough key spellings was implemented and removed for failing that bar -- see :mod:`.backend_compat` for the case that killed it. - -:mod:`.errors` is the other half of the story: it gives FastAPI's own body-validation -failures the same wire shape as these, so a caller sees one error format. """ from .backend_compat import BackendCapability, assert_request_supported, resolve_backend -from .errors import register_validation_error_handler __all__ = [ 'BackendCapability', 'assert_request_supported', - 'register_validation_error_handler', 'resolve_backend', ] diff --git a/src/twinkle/server/validation/backend_compat.py b/src/twinkle/server/validation/backend_compat.py index 82f4ab9e..074bb0a4 100644 --- a/src/twinkle/server/validation/backend_compat.py +++ b/src/twinkle/server/validation/backend_compat.py @@ -29,6 +29,7 @@ from typing import Any, Optional from twinkle.server.exceptions import EndpointUnavailableError, RequestRejectedError +from twinkle.utils.logger import get_logger from twinkle_client.types.base import FieldRole, fields_with_role, read_backend_only @@ -64,6 +65,14 @@ def resolve_backend(service: Any) -> str | None: sampler deployment has no ``backend`` at all. """ backend = getattr(service, 'backend', None) + if backend is None: + # Not an error: Sampler deployments have no ``backend`` attribute at all. But a + # silent ``None`` meant the whole backend-compat preflight vanished with no + # trace, so the skip is now observable. ``debug`` not ``warning``: + # for Sampler the skip is normal and happens every request. + get_logger().debug('backend-compat preflight skipped: %s exposes no ``backend`` attribute', + type(service).__name__) + return None return backend if isinstance(backend, str) else None diff --git a/src/twinkle/server/validation/errors.py b/src/twinkle/server/validation/errors.py deleted file mode 100644 index 895c7eca..00000000 --- a/src/twinkle/server/validation/errors.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) ModelScope Contributors. All rights reserved. -"""Deliver a request-body validation failure in the same shape as every other error. - -FastAPI's default handler answers ``RequestValidationError`` with -``{"detail": [...]}``, which is a second error shape on the wire: a client that learned -to read ``error`` / ``category`` / ``error_code`` from :class:`ErrorPayload` -- what -every other twinkle failure uses -- gets nothing it recognises from a 422. Registering -this on the shared app builder makes the Model, Sampler and Processor deployments answer -identically. - -The per-field ``details`` are the point of a 422: they name the offending field, its -path inside the body, and why it was rejected, so a caller can fix the request without -guessing. There is no traceback -- a rejected body is the caller's problem, not a crash, -and pydantic's error list already localises it exactly. -""" -from __future__ import annotations - -from fastapi import FastAPI, Request -from fastapi.exceptions import RequestValidationError -from fastapi.responses import JSONResponse -from typing import Any - -from twinkle_client.types.errors import ErrorCategory, ErrorPayload - -# A body can produce hundreds of errors (one per element of a mis-typed tensor), and a -# response listing all of them helps nobody while costing bandwidth on every retry. -_MAX_DETAILS = 20 - - -def _detail(error: dict[str, Any]) -> dict[str, Any]: - """One pydantic error as a JSON-safe detail entry.""" - location = [str(part) for part in error.get('loc', ())] - return { - 'field': location[-1] if location else '', - 'path': '.'.join(location), - 'type': error.get('type', ''), - 'message': error.get('msg', ''), - } - - -def _summary(errors: list[dict[str, Any]]) -> str: - fields = [] - for error in errors: - path = '.'.join(str(part) for part in error.get('loc', ())) - if path and path not in fields: - fields.append(path) - shown = ', '.join(fields[:_MAX_DETAILS]) or 'request body' - suffix = '' if len(fields) <= _MAX_DETAILS else f' (+{len(fields) - _MAX_DETAILS} more)' - return f'Request body validation failed for: {shown}{suffix}' - - -def _mentions_unknown_field(errors: list[dict[str, Any]]) -> bool: - return any(error.get('type') == 'extra_forbidden' for error in errors) - - -async def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse: - """Map a body validation failure to a 422 carrying an ``ErrorPayload``.""" - errors = list(exc.errors()) - message = _summary(errors) - if _mentions_unknown_field(errors): - # An unknown top-level field is what an older client looks like against a newer - # server, so say so instead of leaving the caller to infer it from a field list. - message += ('. Unknown fields are rejected; if this worked before, upgrade ' - 'twinkle-kit on the client to match the server version.') - payload = ErrorPayload( - error=message[:1024], - category=ErrorCategory.User, - error_code=422, - request_id=getattr(request.state, 'request_id', None) or '', - details=[_detail(error) for error in errors[:_MAX_DETAILS]], - ) - return JSONResponse(status_code=422, content=payload.model_dump(mode='json', exclude_none=True)) - - -def register_validation_error_handler(app: FastAPI) -> None: - """Install the handler on one deployment app.""" - app.add_exception_handler(RequestValidationError, validation_error_handler) diff --git a/src/twinkle_agentic/async_rl/data_plane.py b/src/twinkle_agentic/async_rl/data_plane.py index da9015f0..5bacc4b9 100644 --- a/src/twinkle_agentic/async_rl/data_plane.py +++ b/src/twinkle_agentic/async_rl/data_plane.py @@ -5,9 +5,10 @@ from typing import Any, Sequence +from twinkle.data_format import REQUIRED_MODEL_INPUT_FIELDS, ROLLOUT_TRAIN_FIELDS, columns_to_tq_fields, rows_to_tq_fields + from .native_tq import (AsyncTQClient, append_fields, batch_size_for_groups, clear_partition, fetch_ready_batch, metadata_size, preallocate_partition, set_sample_tags, split_batch_meta) -from .tq_utils import REQUIRED_MODEL_INPUT_FIELDS, ROLLOUT_TRAIN_FIELDS, columns_to_tq_fields, rows_to_tq_fields from .types import ClaimedBatch, LoraContext, PartitionAdmission, PreparedPartition, PromptGroup, RolloutOutput _REQUIRED_ROLLOUT_FIELDS = frozenset((*REQUIRED_MODEL_INPUT_FIELDS, 'logprobs', 'rewards')) diff --git a/src/twinkle_agentic/async_rl/pipeline.py b/src/twinkle_agentic/async_rl/pipeline.py index 75a78e02..09d5846e 100644 --- a/src/twinkle_agentic/async_rl/pipeline.py +++ b/src/twinkle_agentic/async_rl/pipeline.py @@ -637,7 +637,7 @@ def _train_batch_with_config( *, model_data_parallel_size: int = 1, ) -> dict[str, Any]: - from .tq_utils import REQUIRED_MODEL_INPUT_FIELDS + from twinkle.data_format import REQUIRED_MODEL_INPUT_FIELDS size = int(data.batch_size[0]) inputs = [{name: data[name][index] for name in REQUIRED_MODEL_INPUT_FIELDS} for index in range(size)] diff --git a/src/twinkle_agentic/async_rl/tq_utils.py b/src/twinkle_agentic/async_rl/tq_utils.py deleted file mode 100644 index 55284567..00000000 --- a/src/twinkle_agentic/async_rl/tq_utils.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) ModelScope Contributors. All rights reserved. -from __future__ import annotations - -from twinkle.tq_utils import columns_to_tq_fields, rows_to_tq_fields - -TRANSFORMERS_INPUT_FIELDS = ( - 'input_ids', - 'labels', - 'attention_mask', - 'position_ids', - 'cu_seqlens', - 'completion_mask', - 'pixel_values', - 'image_grid_thw', - 'video_pixel_values', - 'video_grid_thw', - 'input_features', - 'feature_attention_mask', -) -REQUIRED_MODEL_INPUT_FIELDS = ('input_ids', 'labels', 'attention_mask', 'position_ids') -ROLLOUT_TRAIN_FIELDS = (*TRANSFORMERS_INPUT_FIELDS, 'logprobs', 'rewards', 'advantages', 'returns') - -__all__ = [ - 'ROLLOUT_TRAIN_FIELDS', - 'REQUIRED_MODEL_INPUT_FIELDS', - 'TRANSFORMERS_INPUT_FIELDS', - 'columns_to_tq_fields', - 'rows_to_tq_fields', -] diff --git a/src/twinkle_client/types/training.py b/src/twinkle_client/types/training.py index da2c7d67..6cdad789 100644 --- a/src/twinkle_client/types/training.py +++ b/src/twinkle_client/types/training.py @@ -3,7 +3,7 @@ Shared Pydantic models for twinkle training runs and checkpoints. These types are used both by twinkle_client (as request/response shapes) -and by twinkle.server.common.io_utils (as persistence models). +and by twinkle.server.checkpoint.twinkle (as persistence / response models). """ from datetime import datetime from pydantic import BaseModel diff --git a/tests/data_format/test_tq_fields.py b/tests/data_format/test_tq_fields.py new file mode 100644 index 00000000..3d135548 --- /dev/null +++ b/tests/data_format/test_tq_fields.py @@ -0,0 +1,58 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Tests for ``twinkle.data_format.tq_fields``. + +The packing helpers carry field-consistency validation and a numeric/non-numeric +type branch but previously had zero test coverage. These cover the three branches +required by (empty rows, inconsistent fields, numeric+non-numeric mix), plus a +subprocess assertion (not depending on the ``async-rl`` extra) that guards: the +module must import cleanly without ``tensordict``. +""" +import subprocess +import sys + +import pytest + +tensordict = pytest.importorskip('tensordict') + + +def test_rows_to_tq_fields_empty_rows(): + from twinkle.data_format import rows_to_tq_fields + + packed = rows_to_tq_fields([]) + assert packed.batch_size[0] == 0 + + +def test_rows_to_tq_fields_rejects_inconsistent_fields(): + """Rows with differing key sets must raise, not silently pack a ragged TensorDict.""" + from twinkle.data_format import rows_to_tq_fields + + with pytest.raises(ValueError): + rows_to_tq_fields([{'input_ids': [1]}, {'input_ids': [2], 'labels': [3]}]) + + +def test_columns_to_tq_fields_mixes_numeric_and_non_numeric(): + """Numeric columns go through ``torch.tensor``; the rest through ``NonTensorStack``.""" + import torch + + from twinkle.data_format import columns_to_tq_fields + + packed = columns_to_tq_fields({'scores': [1, 2], 'names': ['a', 'b']}, 2) + assert packed.batch_size[0] == 2 + assert isinstance(packed['scores'], torch.Tensor) + assert list(packed['names']) == ['a', 'b'] + + +def test_tq_fields_imports_without_tensordict(): + """The module must import cleanly without the async-rl extra installed. + + Subprocess with ``tensordict`` blocked from ``sys.modules``, asserting that importing + the module (and reading the constants) does not touch it -- function-level imports are + what make that true, so hoisting them would break this. + """ + code = ( + 'import sys;' + "sys.modules['tensordict'] = None;" + 'from twinkle.data_format import tq_fields, ROLLOUT_TRAIN_FIELDS;' + "assert 'input_ids' in ROLLOUT_TRAIN_FIELDS" + ) + subprocess.run([sys.executable, '-c', code], check=True) diff --git a/tests/infra/test_ray_get_timeout.py b/tests/infra/test_ray_get_timeout.py index 00a3b79c..e82bfc44 100644 --- a/tests/infra/test_ray_get_timeout.py +++ b/tests/infra/test_ray_get_timeout.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Unit tests for the Sync_Dispatch_Path time bound (spec T1.6 / R9#1). +"""Unit tests for the Sync_Dispatch_Path time bound. These exercise only ``twinkle.infra`` against a plain sleeping Ray actor. They depend on neither GPU, Megatron, nor any ``src/twinkle/server/**`` component. @@ -54,7 +54,7 @@ def _make_driver(): def test_execute_all_sync_times_out(_ray_and_ray_mode): - """R9#1: execute_all_sync(timeout=) raises when the remote does not return in time.""" + """``execute_all_sync(timeout=)`` raises when the remote does not return in time.""" actor = _Sleeper.remote() workers_and_args = [(actor, [3.0], {})] with pytest.raises(ray.exceptions.GetTimeoutError): @@ -125,7 +125,7 @@ def slow_batch(self, seconds): def test_decorator_timeout_zero_is_not_treated_as_falsy(): - """timeout=0 means 'time out immediately', not 'fall back to unbounded'.""" + """Timeout=0 means 'time out immediately', not 'fall back to unbounded'.""" def slow(self, seconds): return seconds diff --git a/tests/server/conftest.py b/tests/server/conftest.py index de4644de..dfcc7f2e 100644 --- a/tests/server/conftest.py +++ b/tests/server/conftest.py @@ -13,7 +13,7 @@ before each test function. Tests that pin a non-default ``key_prefix`` get their own actor; this fixture intentionally leaves those alone. -Evidence boundary (spec T8.4 / R9#10): every mock-model backend method accepts +Evidence boundary: every mock-model backend method accepts ``**kwargs`` without argument validation, and the mock enters no real collective. A mock-backed test therefore proves neither request/argument validation nor NCCL behavior (asymmetric failure, collective mis-pairing, ReduceScatter, etc.). It may diff --git a/tests/server/contract/client_api_harness.py b/tests/server/contract/client_api_harness.py index 975b579c..ab82e058 100644 --- a/tests/server/contract/client_api_harness.py +++ b/tests/server/contract/client_api_harness.py @@ -55,23 +55,23 @@ def build_data_plane_app() -> FastAPI: def build_gateway_app() -> FastAPI: from twinkle.server.gateway.openai_handlers import _register_openai_routes - from twinkle.server.gateway.tinker_handlers import _register_tinker_routes - from twinkle.server.gateway.twinkle_handlers import _register_twinkle_routes + from twinkle.server.gateway.tinker_handlers import _register_gateway_tinker_routes + from twinkle.server.gateway.twinkle_handlers import _register_gateway_twinkle_routes app = FastAPI() - _register_tinker_routes(app, _noop_self) - _register_twinkle_routes(app, _noop_self) + _register_gateway_tinker_routes(app, _noop_self) + _register_gateway_twinkle_routes(app, _noop_self) _register_openai_routes(app, _noop_self) return app def build_model_app() -> FastAPI: - from twinkle.server.model.tinker_handlers import _register_tinker_routes - from twinkle.server.model.twinkle_handlers import _register_twinkle_routes + from twinkle.server.model.tinker_handlers import _register_model_tinker_routes + from twinkle.server.model.twinkle_handlers import _register_model_twinkle_routes app = FastAPI() - _register_tinker_routes(app, _noop_self) - _register_twinkle_routes(app, _noop_self) + _register_model_tinker_routes(app, _noop_self) + _register_model_twinkle_routes(app, _noop_self) return app diff --git a/tests/server/contract/test_client_api_contract.py b/tests/server/contract/test_client_api_contract.py index 15e6cfb6..065017cf 100644 --- a/tests/server/contract/test_client_api_contract.py +++ b/tests/server/contract/test_client_api_contract.py @@ -54,8 +54,8 @@ def test_full_surface_extraction_is_self_consistent(): def test_schedule_task_and_wait_removed(): # server-request-lifecycle (Part 2) deletes the in-process blocking wait: the - # future record is now the single delivery channel (Property 1). - from twinkle.server.utils.task_queue.mixin import TaskQueueMixin + # future record is now the single delivery channel. + from twinkle.server.task_queue.mixin import TaskQueueMixin assert not hasattr(TaskQueueMixin, 'schedule_task_and_wait') assert hasattr(TaskQueueMixin, 'submit_and_peek') diff --git a/tests/server/contract/test_error_wire.py b/tests/server/contract/test_error_wire.py index 79ca7785..41b394e4 100644 --- a/tests/server/contract/test_error_wire.py +++ b/tests/server/contract/test_error_wire.py @@ -4,7 +4,7 @@ from fastapi.testclient import TestClient from tinker.types import RequestFailedResponse -from twinkle.server.gateway.tinker_handlers import _register_tinker_routes +from twinkle.server.gateway.tinker_handlers import _register_gateway_tinker_routes class _State: @@ -26,7 +26,7 @@ class _Gateway: def test_retrieve_future_returns_parseable_error_payload(): app = FastAPI() - _register_tinker_routes(app, lambda: _Gateway()) + _register_gateway_tinker_routes(app, lambda: _Gateway()) response = TestClient(app).post('/retrieve_future', json={'request_id': 'req-1'}) diff --git a/tests/server/gateway/test_openai_handlers.py b/tests/server/gateway/test_openai_handlers.py index 8a98afbb..98f8bf19 100644 --- a/tests/server/gateway/test_openai_handlers.py +++ b/tests/server/gateway/test_openai_handlers.py @@ -17,14 +17,6 @@ # ---------- Fixtures ------------------------------------------------------- # -@pytest.fixture(autouse=True) -def _reset_template_cache(): - from twinkle.server.gateway.openai_handlers import _template_initialized - _template_initialized.clear() - yield - _template_initialized.clear() - - @pytest.fixture def mock_gateway(): """Build a minimal FastAPI app with OpenAI routes and a mock GatewayServer.""" @@ -42,7 +34,10 @@ def mock_gateway(): mock_self.state = mock_state mock_self.proxy = mock_proxy mock_self.supported_models = [types.SupportedModel(model_name='Qwen/Qwen3.5-4B')] - mock_self._supported_model_names = frozenset(['Qwen/Qwen3.5-4B']) + mock_self.supported_model_names = frozenset(['Qwen/Qwen3.5-4B']) + # Per-instance template cache; a fresh mock per test isolates it, so the + # former module-global clear fixture is no longer needed. + mock_self._template_initialized = set() app = FastAPI() _register_openai_routes(app, lambda: mock_self) @@ -125,7 +120,7 @@ def test_missing_messages_returns_400(self, mock_gateway): def test_model_not_found_returns_404(self, mock_gateway): mock_self, app = mock_gateway mock_self.supported_models = [] # No supported models - mock_self._supported_model_names = frozenset() + mock_self.supported_model_names = frozenset() client = TestClient(app) resp = client.post( diff --git a/tests/server/gateway/test_proxy.py b/tests/server/gateway/test_proxy.py new file mode 100644 index 00000000..e5000cf1 --- /dev/null +++ b/tests/server/gateway/test_proxy.py @@ -0,0 +1,66 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Directed tests for ``gateway/proxy.py``. + +Covers the route-URL construction now sourced from ``gateway.routes``, the +``H_MULTIPLEX`` header compatibility in ``_prepare_headers``, and the 502 ``ErrorPayload`` +fallback when the upstream is unreachable. +""" +from __future__ import annotations + +import json + +import pytest +from starlette.requests import Request +from unittest.mock import AsyncMock + +from twinkle.server.gateway.proxy import ServiceProxy +from twinkle_client.http.headers import H_MULTIPLEX, H_MULTIPLEX_LEGACY, H_REQUEST_ID + + +def _make_request(headers: list[tuple[bytes, bytes]] | None = None) -> Request: + scope = { + 'type': 'http', + 'method': 'POST', + 'headers': headers or [], + 'query_string': b'', + 'path': '/', + } + return Request(scope) + + +@pytest.mark.parametrize( + 'route_prefix,host,expected', + [ + ('/api/v1', 'localhost', 'http://localhost:8000/api/v1/model/Qwen/tinker/forward'), + ('/api/v1/', 'localhost', 'http://localhost:8000/api/v1/model/Qwen/tinker/forward'), + ('', 'localhost', 'http://localhost:8000/model/Qwen/tinker/forward'), + ('/api/v1', '0.0.0.0', 'http://localhost:8000/api/v1/model/Qwen/tinker/forward'), + ], +) +def test_build_target_url(route_prefix, host, expected): + proxy = ServiceProxy(http_options={'host': host, 'port': 8000}, route_prefix=route_prefix) + assert proxy._build_target_url('model', 'Qwen', 'tinker/forward') == expected + + +def test_prepare_headers_sets_multiplex_from_request_id(): + proxy = ServiceProxy(http_options={}, route_prefix='/api/v1') + headers = proxy._prepare_headers({H_REQUEST_ID: 'req-1'}) + assert headers.get(H_MULTIPLEX) == 'req-1' + assert headers.get(H_MULTIPLEX_LEGACY) == 'req-1' + # ``host`` / ``content-length`` are stripped before forwarding. + assert 'host' not in {k.lower() for k in headers} + + +@pytest.mark.asyncio +async def test_proxy_request_502_fallback_returns_error_payload(): + proxy = ServiceProxy(http_options={'host': 'localhost', 'port': 8000}, route_prefix='/api/v1') + proxy.client.request = AsyncMock(side_effect=RuntimeError('upstream down')) + request = _make_request(headers=[(H_REQUEST_ID.encode(), b'req-9')]) + + response = await proxy.proxy_request(request, 'tinker/forward', 'Qwen', 'model', body_override=b'{}') + + assert response.status_code == 502 + payload = json.loads(response.body) + assert payload['error_code'] == 502 + assert payload['category'] == 'server' + assert payload['request_id'] == 'req-9' diff --git a/tests/server/integration/test_actor_recovery.py b/tests/server/integration/test_actor_recovery.py index 64696637..16527bba 100644 --- a/tests/server/integration/test_actor_recovery.py +++ b/tests/server/integration/test_actor_recovery.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Post-timeout liveness probe and health status bit (T4.2 / R3#2-3). +"""Post-timeout liveness probe and health status bit. Binds the real ``ModelManagement`` health methods onto a minimal harness with a toggleable mock ``ping`` and a direct ``call_backend``. No GPU/Ray/full server. @@ -10,7 +10,7 @@ from fastapi import FastAPI from twinkle.server.model.app import ModelManagement -from twinkle.server.model.twinkle_handlers import _register_twinkle_routes +from twinkle.server.model.twinkle_handlers import _register_model_twinkle_routes class _MockModel: @@ -67,7 +67,7 @@ async def test_health_route_returns_503_when_probe_fails(): model.alive = False harness = _HealthHarness(model) app = FastAPI() - _register_twinkle_routes(app, lambda: harness) + _register_model_twinkle_routes(app, lambda: harness) route = next(route for route in app.routes if getattr(route, 'path', None) == '/healthz') response = await route.endpoint(object(), harness) diff --git a/tests/server/integration/test_blocking_boundary.py b/tests/server/integration/test_blocking_boundary.py index 93732604..7d35e229 100644 --- a/tests/server/integration/test_blocking_boundary.py +++ b/tests/server/integration/test_blocking_boundary.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Blocking_Call_Boundary integration tests (T3.8 / R9#2 / Property 3-4). +"""Blocking_Call_Boundary integration tests. These exercise the real ``TaskQueueMixin.call_backend`` through a minimal harness that sets only the two attributes it uses (a dedicated executor and the optional @@ -19,27 +19,34 @@ ray = pytest.importorskip('ray') -from twinkle.server.utils.task_queue.mixin import TaskQueueMixin # noqa: E402 -from twinkle.server.utils.task_queue.types import BackendBusyError # noqa: E402 +from twinkle.server.task_queue.backend_gate import BackendGate # noqa: E402 +from twinkle.server.task_queue.mixin import TaskQueueMixin # noqa: E402 +from twinkle.server.task_queue.types import BackendBusyError # noqa: E402 class _Harness(TaskQueueMixin): """Minimal holder exposing the real call_backend with a chosen gate setting.""" def __init__(self, gate_enabled: bool, *, max_workers: int | None = None) -> None: - self._backend_executor = ThreadPoolExecutor(max_workers=max_workers, thread_name_prefix='twinkle-backend') - self._backend_probe_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix='twinkle-backend-probe') - self._backend_admission = asyncio.Lock() if gate_enabled else None - self._backend_poisoned = asyncio.Event() + # ``call_backend`` delegates to the extracted ``BackendGate``; build one and + # alias its internals so the assertions below still read the same names. + self._backend_gate = BackendGate(enable_admission_gate=gate_enabled) + if max_workers is not None: + self._backend_gate._executor.shutdown(wait=False) + self._backend_gate._executor = ThreadPoolExecutor( + max_workers=max_workers, thread_name_prefix='twinkle-backend') + self._backend_executor = self._backend_gate._executor + self._backend_probe_executor = self._backend_gate._probe_executor + self._backend_admission = self._backend_gate._admission + self._backend_poisoned = self._backend_gate._poisoned def close(self) -> None: - self._backend_executor.shutdown(wait=False, cancel_futures=True) - self._backend_probe_executor.shutdown(wait=False, cancel_futures=True) + self._backend_gate.shutdown() @pytest.mark.asyncio async def test_healthz_style_probe_responsive_during_slow_backend(): - """Property 3: while a slow backend call is in flight, an admit=False probe + """While a slow backend call is in flight, an admit=False probe (as /healthz uses) returns well within 5 seconds.""" h = _Harness(gate_enabled=True) try: @@ -116,7 +123,7 @@ async def test_cancelled_queued_backend_call_releases_gate(): @pytest.mark.asyncio async def test_gate_held_by_leaked_call_fast_fails_next_task(): - """Property 4 / R2#4: a call that outlives its wait_for keeps the gate; the next + """A call that outlives its wait_for keeps the gate; the next admitting call fails fast with BackendBusyError instead of entering the backend.""" h = _Harness(gate_enabled=True) entered = {'count': 0} @@ -190,7 +197,7 @@ async def healthz(): @pytest.mark.asyncio async def test_sampler_without_gate_runs_two_calls_concurrently(): - """R9#2 case 3 / opt-in: with the gate disabled (SamplerManagement), two backend + """Case 3 / opt-in: with the gate disabled (SamplerManagement), two backend calls are in flight at once rather than serialized.""" h = _Harness(gate_enabled=False) try: diff --git a/tests/server/integration/test_dpo_e2e.py b/tests/server/integration/test_dpo_e2e.py index 035ae188..7a69cbca 100644 --- a/tests/server/integration/test_dpo_e2e.py +++ b/tests/server/integration/test_dpo_e2e.py @@ -182,7 +182,7 @@ def test_dpo_tinker(): """ from tinker import types from twinkle.dataloader import DataLoader - from twinkle.server.common import input_feature_to_datum + from twinkle.server.model.tinker_datum import input_feature_to_datum backend = get_backend() log(f'=== test_dpo_tinker [backend={backend}] ===') diff --git a/tests/server/integration/test_full_param_e2e.py b/tests/server/integration/test_full_param_e2e.py index b328b8a5..506a050b 100644 --- a/tests/server/integration/test_full_param_e2e.py +++ b/tests/server/integration/test_full_param_e2e.py @@ -45,7 +45,7 @@ from twinkle.dataloader import DataLoader # noqa: E402 from twinkle.dataset import Dataset, DatasetMeta # noqa: E402 from twinkle.preprocessor import SelfCognitionProcessor # noqa: E402 -from twinkle.server.common import input_feature_to_datum # noqa: E402 +from twinkle.server.model.tinker_datum import input_feature_to_datum # noqa: E402 init_tinker_client() diff --git a/tests/server/integration/test_nccl_safe_tinker_e2e.py b/tests/server/integration/test_nccl_safe_tinker_e2e.py index 3d84e9fe..6b8bc3e0 100644 --- a/tests/server/integration/test_nccl_safe_tinker_e2e.py +++ b/tests/server/integration/test_nccl_safe_tinker_e2e.py @@ -32,9 +32,9 @@ EXECUTION_TIMEOUT = float(os.environ.get('TWINKLE_TEST_EXECUTION_TIMEOUT', '30')) TIMEOUT = EXECUTION_TIMEOUT + 15 # The `global_rank=` attribution is added by `nccl_safe_megatron`, which decorates -# only the Megatron backend; the transformers backend carries no such annotation -# (its former silent-degradation decorator was removed by R6#3). Gate the rank-attribution assertion -# on the backend so this file is safe under TWINKLE_TEST_BACKEND=transformers. +# only the Megatron backend; the transformers backend carries no such annotation. +# Gate the rank-attribution assertion on the backend so this file is safe under +# TWINKLE_TEST_BACKEND=transformers. BACKEND = os.environ.get('TWINKLE_TEST_BACKEND', 'megatron') @@ -90,7 +90,7 @@ def test_failure_is_terminal_then_valid_request_succeeds(): Replaces the former assertion "failure degraded to zero loss and training continued". If the recovery request does not reach a terminal success, that is - recorded as evidence that R3#2-3 actor recovery and R2#3-4 admission gate are + recorded as evidence that actor recovery and admission gate are necessary, not optional. """ from tinker import types @@ -106,7 +106,7 @@ def test_failure_is_terminal_then_valid_request_succeeds(): assert time.time() - start < TIMEOUT, 'malformed request must fail fast, not hang (NCCL)' # Megatron must recover successfully. Tinker's Transformers path executes - # forward/loss/backward separately; after a mid-iteration failure, R6#14 only + # forward/loss/backward separately; after a mid-iteration failure, the test only # guarantees that the next request reaches a terminal state. _assert_recovery_terminal(tc) @@ -123,7 +123,7 @@ def test_partial_rank_failure_is_terminal_then_recovers(): tc.forward_backward(batch, 'importance_sampling').result(timeout=TIMEOUT) assert caught.value.category is types.RequestErrorCategory.Server # Megatron attributes the failure to a global rank via nccl_safe_megatron; the - # transformers backend has no such annotation (R6#3 removed its old decorator). + # transformers backend has no such annotation ( removed its old decorator). if BACKEND == 'megatron': assert 'global_rank=' in str(caught.value) assert time.time() - start < TIMEOUT diff --git a/tests/server/integration/test_nccl_safe_twinkle_e2e.py b/tests/server/integration/test_nccl_safe_twinkle_e2e.py index 29b89be4..5effc140 100644 --- a/tests/server/integration/test_nccl_safe_twinkle_e2e.py +++ b/tests/server/integration/test_nccl_safe_twinkle_e2e.py @@ -33,9 +33,8 @@ ADAPTER_NAME = 'loud-failure-test' # The `global_rank=` attribution is added by `nccl_safe_megatron`, which decorates # only the Megatron backend; the transformers backend's forward_backward carries no -# such annotation (its former silent-degradation decorator was removed by R6#3). Gate the -# rank-attribution assertion on the backend so this file is safe to run under the -# integration-e2e SKILL's TWINKLE_TEST_BACKEND=transformers path. +# such annotation. Gate the rank-attribution assertion on the backend so this file is +# safe to run under the integration-e2e SKILL's TWINKLE_TEST_BACKEND=transformers path. BACKEND = os.environ.get('TWINKLE_TEST_BACKEND', 'megatron') @@ -83,7 +82,7 @@ def test_failure_is_terminal_then_valid_request_succeeds(): Replaces the former assertion "failure degraded to zero loss and training continued". If the recovery request does not reach a terminal success, that is - recorded as evidence that R3#2-3 actor recovery and R2#3-4 admission gate are + recorded as evidence that actor recovery and admission gate are necessary, not optional. """ model = _init_client() diff --git a/tests/server/integration/test_sft_e2e.py b/tests/server/integration/test_sft_e2e.py index 61f0bd3d..16a014b8 100644 --- a/tests/server/integration/test_sft_e2e.py +++ b/tests/server/integration/test_sft_e2e.py @@ -156,7 +156,7 @@ def test_sft_tinker(): """ from tinker import types from twinkle.dataloader import DataLoader - from twinkle.server.common import input_feature_to_datum + from twinkle.server.model.tinker_datum import input_feature_to_datum backend = get_backend() log(f'=== test_sft_tinker [backend={backend}] ===') diff --git a/tests/server/lifecycle/test_envelope.py b/tests/server/lifecycle/test_envelope.py index 2c2216f7..fdc588bd 100644 --- a/tests/server/lifecycle/test_envelope.py +++ b/tests/server/lifecycle/test_envelope.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Unit tests for the single FutureRecord -> TaskEnvelope mapping point (T1.2).""" +"""Unit tests for the single FutureRecord -> TaskEnvelope mapping point.""" from __future__ import annotations import pytest @@ -21,7 +21,7 @@ def test_wire_maps_cover_exactly_the_canonical_reason_codes(): def test_completed_with_none_result_is_a_success_not_a_failure(): - """R1#4 / Property 4: `completed` + `result is None` is a valid success.""" + """`completed` + `result is None` is a valid success.""" env = envelope_from_record('req-1', {'status': 'completed', 'result': None}) assert env.status == 'completed' assert env.result is None diff --git a/tests/server/lifecycle/test_envelope_coverage.py b/tests/server/lifecycle/test_envelope_coverage.py index c6ece290..bf070af8 100644 --- a/tests/server/lifecycle/test_envelope_coverage.py +++ b/tests/server/lifecycle/test_envelope_coverage.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Task_Envelope coverage check (T5.4 / R5#7, R5#2/#3). +"""Task_Envelope coverage check. Walks the model and sampler route tables and asserts that every twinkle-native POST route that enters the Task_Queue declares ``response_model = TaskEnvelope``. @@ -13,7 +13,7 @@ from tests.server.contract.client_api_harness import build_model_app, build_sampler_app from twinkle_client.types.lifecycle import TaskEnvelope -# The single exemption declaration (R5#7), keyed BY APP. A flat path set would be wrong: +# The single exemption declaration, keyed BY APP. A flat path set would be wrong: # ``/twinkle/set_template`` and ``/twinkle/apply_patch`` exist on both apps, but only the # sampler's bypass the queue -- the model's are queued and must return a Task_Envelope. # Sharing one set silently exempted the model's two and left a hole in this guard. @@ -29,7 +29,7 @@ '/twinkle/add_adapter_to_sampler', '/twinkle/apply_patch', '/twinkle/unload_adapter_paths', - # the one streaming exception (R5#2) + # the one streaming exception '/twinkle/sample_stream', }, } diff --git a/tests/server/lifecycle/test_preflight_rejection.py b/tests/server/lifecycle/test_preflight_rejection.py index 4b17e9e6..0c481ee8 100644 --- a/tests/server/lifecycle/test_preflight_rejection.py +++ b/tests/server/lifecycle/test_preflight_rejection.py @@ -1,8 +1,8 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Decision_Boundary tests: preflight rejects with real status codes and zero writes. -Covers Property 3 / R8#3 (a rejected request writes no future record) and the -TwinkleServerError handler wire shape (R3#5/#6). No Ray or GPU is involved: the +Covers the case where a rejected request writes no future record, plus the +TwinkleServerError handler wire shape. No Ray or GPU is involved: the task queue is driven with a spy state that counts ``store_future_status`` calls. """ from __future__ import annotations @@ -14,8 +14,8 @@ from twinkle.server.deployment import twinkle_server_error_handler from twinkle.server.exceptions import (BatchSizeError, InputTokensExceededError, RateLimitExceededError, RequestRejectedError, TwinkleServerError) -from twinkle.server.utils.task_queue.config import TaskQueueConfig -from twinkle.server.utils.task_queue.mixin import TaskQueueMixin +from twinkle.server.task_queue.config import TaskQueueConfig +from twinkle.server.task_queue.mixin import TaskQueueMixin class _SpyState: @@ -45,7 +45,7 @@ async def _noop(): @pytest.mark.asyncio async def test_input_tokens_rejection_is_422_and_zero_writes(): - """Property 3 / R8#3: an over-limit request raises 422 and writes no record.""" + """An over-limit request raises 422 and writes no record.""" h = _Harness(enabled=True, max_input_tokens=10) try: with pytest.raises(InputTokensExceededError) as exc: @@ -88,7 +88,7 @@ async def test_rate_limit_rejection_is_429_and_zero_writes(): @pytest.mark.asyncio async def test_disabled_queue_skips_preflight(): - """The 'no token or queue disabled' short circuit is preserved (R3#3).""" + """The 'no token or queue disabled' short circuit is preserved.""" h = _Harness(enabled=False, max_input_tokens=10) try: ref = await h.schedule_task(lambda: _noop(), model_id='m', token='tok', input_tokens=999, task_type='forward') @@ -98,7 +98,7 @@ async def test_disabled_queue_skips_preflight(): def test_error_handler_puts_fields_at_top_level(): - """R3#5/#6: the handler returns error_code as the status and fields at top level.""" + """The handler returns error_code as the status and fields at top level.""" app = FastAPI() app.add_exception_handler(TwinkleServerError, twinkle_server_error_handler) diff --git a/tests/server/lifecycle/test_protocols.py b/tests/server/lifecycle/test_protocols.py new file mode 100644 index 00000000..4f670a5b --- /dev/null +++ b/tests/server/lifecycle/test_protocols.py @@ -0,0 +1,26 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""``run_submit`` must stay a free function. + +Reverting it to a decorator would rewrite its signature and deepen the route graph, which +once hit the CPython C-stack recursion limit in ``serve.ingress``'s cloudpickle phase. +Adding a ``self: QueuedDeployment`` annotation is a zero-runtime-cost change, so +the function form must be unchanged. +""" +import inspect + +from twinkle.server.lifecycle.protocols import DataParallelDeployment, QueuedDeployment +from twinkle.server.lifecycle.submit import input_metrics, run_submit + + +def test_run_submit_is_a_free_function(): + assert inspect.isfunction(run_submit) + assert inspect.iscoroutinefunction(run_submit) + assert inspect.isfunction(input_metrics) + + +def test_host_protocols_are_two_layers(): + # DataParallelDeployment is the strictly stronger contract (adds data_world_size). + # (``issubclass`` is avoided: runtime_checkable Protocols with data members raise.) + assert QueuedDeployment in DataParallelDeployment.__mro__ + assert hasattr(DataParallelDeployment, 'data_world_size') + assert not hasattr(QueuedDeployment, 'data_world_size') diff --git a/tests/server/lifecycle/test_retrieve_endpoint.py b/tests/server/lifecycle/test_retrieve_endpoint.py index fcc0d6e7..84b8767e 100644 --- a/tests/server/lifecycle/test_retrieve_endpoint.py +++ b/tests/server/lifecycle/test_retrieve_endpoint.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Wire tests for the twinkle Retrieve_Endpoint (T1.4, Property 4/5, R8#4/#5). +"""Wire tests for the twinkle Retrieve_Endpoint. These use a fake state and FastAPI's TestClient; no Ray runtime is needed, so they live outside the state-actor fixtures. @@ -13,7 +13,7 @@ from twinkle.server.deployment import twinkle_server_error_handler from twinkle.server.exceptions import TwinkleServerError -from twinkle.server.gateway.twinkle_handlers import _register_twinkle_routes +from twinkle.server.gateway.twinkle_handlers import _register_gateway_twinkle_routes class _State: @@ -35,12 +35,12 @@ def __init__(self, record): def _client(record) -> TestClient: app = FastAPI() app.add_exception_handler(TwinkleServerError, twinkle_server_error_handler) - _register_twinkle_routes(app, lambda: _Gateway(record)) + _register_gateway_twinkle_routes(app, lambda: _Gateway(record)) return TestClient(app) def test_completed_with_null_result_returns_200_and_null(monkeypatch): - """Property 4 / R8#4: completed + result=None is 200 with result null, not 500.""" + """Completed + result=None is 200 with result null, not 500.""" client = _client({'status': 'completed', 'result': None}) resp = client.post('/twinkle/retrieve_future', json={'request_id': 'req-1'}) assert resp.status_code == 200 @@ -71,7 +71,7 @@ def test_domain_failure_returns_200_and_valid_envelope(): def test_always_missing_record_404s_only_after_the_full_window(monkeypatch): - """R2#5: a request_id that never appears returns 404, and only after waiting a window.""" + """A request_id that never appears returns 404, and only after waiting a window.""" monkeypatch.setenv('TWINKLE_LONG_POLL_TIMEOUT', '0.3') client = _client(None) start = time.monotonic() diff --git a/tests/server/lifecycle/test_run_submit_dedup.py b/tests/server/lifecycle/test_run_submit_dedup.py index f9038d55..0b16f6c4 100644 --- a/tests/server/lifecycle/test_run_submit_dedup.py +++ b/tests/server/lifecycle/test_run_submit_dedup.py @@ -38,7 +38,7 @@ async def release_seq(self, dedup_key): class _FakeManagement: def __init__(self, record_after_claim): self.state = _FakeState(record_after_claim) - self._task_queue_config = SimpleNamespace(effective_execution_timeout=60.0) + self.task_queue_config = SimpleNamespace(effective_execution_timeout=60.0) # A real deployment declares its backend; preflight reads it from here. self.backend = 'transformers' self.data_world_size = 1 diff --git a/tests/server/lifecycle/test_static_guards.py b/tests/server/lifecycle/test_static_guards.py index 18f280e8..75c1af54 100644 --- a/tests/server/lifecycle/test_static_guards.py +++ b/tests/server/lifecycle/test_static_guards.py @@ -1,9 +1,9 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Static / structural guards for the lifecycle refactor. -- Property 1 (R8#7): the deleted symbols occur zero times under ``src/twinkle/**``. -- Property 0: ``TaskEnvelope`` has exactly one construction site. -- D5 client-side invariant (R1#12): the client HTTP timeout is <= 120 and strictly +- the deleted symbols occur zero times under ``src/twinkle/**``. +- ``TaskEnvelope`` has exactly one construction site. +- The client-side invariant: the client HTTP timeout is <= 120 and strictly greater than the server Long_Poll_Window. - The task status set has two independent declarations that must not drift. @@ -20,7 +20,7 @@ _SRC = _REPO_ROOT / 'src' / 'twinkle' # Symbols the refactor removed. A wildcard search (not a per-file list) must find -# each of them zero times across the whole server tree (R8#7, Property 1). +# each of them zero times across the whole server tree. _FORBIDDEN_SYMBOLS = ( 'schedule_task_and_wait', 'run_task', @@ -49,7 +49,7 @@ def test_client_http_timeout_bounds(): def test_task_envelope_has_exactly_one_construction_site(): - """Property 0's structural precondition: one mapping point, mechanically enforced. + """'s structural precondition: one mapping point, mechanically enforced. ``envelope_from_record`` is the only place a FutureRecord becomes a TaskEnvelope, so ``failed`` always lands in ``error`` and never in ``result`` regardless of which @@ -84,7 +84,7 @@ def test_server_task_status_enum_matches_client_literal(): """ from typing import get_args - from twinkle.server.utils.task_queue.types import TaskStatus as ServerTaskStatus + from twinkle.server.task_queue.types import TaskStatus as ServerTaskStatus from twinkle_client.types.lifecycle import TERMINAL_STATUSES from twinkle_client.types.lifecycle import TaskStatus as WireTaskStatus diff --git a/tests/server/lifecycle/test_submit_peek_e2e.py b/tests/server/lifecycle/test_submit_peek_e2e.py index 0009a64c..cf05057e 100644 --- a/tests/server/lifecycle/test_submit_peek_e2e.py +++ b/tests/server/lifecycle/test_submit_peek_e2e.py @@ -1,11 +1,10 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""End-to-end proof of the Inline_Fast_Path + Client_Future_Layer seam (T2.3). +"""End-to-end proof of the Inline_Fast_Path + Client_Future_Layer seam. A minimal harness drives the *real* ``submit_and_peek`` against a real compute worker and a real (memory) ServerState -- no HTTP, no GPU. The resulting envelope is round-tripped through model_dump/model_validate (simulating the wire) and fed to -the real client ``resolve``, so this covers the exact submit -> client path that -Property 0 protects. +the real client ``resolve``, so this covers the exact submit -> client path. """ from __future__ import annotations @@ -14,8 +13,8 @@ ray = pytest.importorskip('ray') from twinkle.server.state import ServerState # noqa: E402 -from twinkle.server.utils.task_queue.config import TaskQueueConfig # noqa: E402 -from twinkle.server.utils.task_queue.mixin import TaskQueueMixin # noqa: E402 +from twinkle.server.task_queue.config import TaskQueueConfig # noqa: E402 +from twinkle.server.task_queue.mixin import TaskQueueMixin # noqa: E402 from twinkle_client import _future # noqa: E402 from twinkle_client.exceptions import TaskFailedError # noqa: E402 from twinkle_client.types.lifecycle import TaskEnvelope # noqa: E402 @@ -39,7 +38,7 @@ def _across_the_wire(env: TaskEnvelope) -> TaskEnvelope: @pytest.mark.asyncio async def test_window_completed_task_is_single_round_trip(monkeypatch): - """R8#1: a task terminal within the window makes the client issue zero retrieves.""" + """A task terminal within the window makes the client issue zero retrieves.""" h = _Harness() async def _ok(): @@ -59,7 +58,7 @@ async def _ok(): @pytest.mark.asyncio async def test_window_failed_task_surfaces_payload_as_taskfailed(monkeypatch): - """R8#2 / Property 0: a failure inside the window reaches the client via the submit + """A failure inside the window reaches the client via the submit response and is raised as TaskFailedError with its payload intact.""" h = _Harness() diff --git a/tests/server/lifecycle/test_timing_bounds.py b/tests/server/lifecycle/test_timing_bounds.py index 7d484977..b2f5f546 100644 --- a/tests/server/lifecycle/test_timing_bounds.py +++ b/tests/server/lifecycle/test_timing_bounds.py @@ -1,11 +1,11 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Timing guards for the lifecycle constants. -- Property 2 (R1#6): a single Submit_Endpoint's server-side duration is bounded by the +- a single Submit_Endpoint's server-side duration is bounded by the Inline_Fast_Path window + 1s and is INDEPENDENT of how long the task itself runs. This is the spec's core benefit claim and was previously the only property with no automated guard. -- R2#4 / R2#6 / D4: the retrieve poll interval is a single shared declaration, is strictly +- The retrieve poll interval is a single shared declaration, is strictly inside the Long_Poll_Window, and is deliberately FIXED (see the measurement recorded in ``poll_config`` and in the test below). """ @@ -21,8 +21,8 @@ from twinkle.server.lifecycle.poll_config import long_poll_window, retrieve_poll_interval # noqa: E402 from twinkle.server.state import ServerState # noqa: E402 -from twinkle.server.utils.task_queue.config import TaskQueueConfig # noqa: E402 -from twinkle.server.utils.task_queue.mixin import TaskQueueMixin # noqa: E402 +from twinkle.server.task_queue.config import TaskQueueConfig # noqa: E402 +from twinkle.server.task_queue.mixin import TaskQueueMixin # noqa: E402 _WINDOW = 0.05 @@ -37,7 +37,7 @@ def __init__(self) -> None: @pytest.mark.asyncio async def test_submit_duration_is_bounded_and_task_duration_independent(): - """Property 2: submit returns on the window, not on task completion.""" + """Submit returns on the window, not on task completion.""" h = _Harness() async def fast(): @@ -58,7 +58,7 @@ async def slow(): env = await h.submit_and_peek(lambda: slow(), task_type='forward_backward') elapsed = time.monotonic() - started - # Bounded by the window + 1s even though the task needs 5s (R1#6). + # Bounded by the window + 1s even though the task needs 5s. assert elapsed < _WINDOW + 1.0, f'submit took {elapsed:.3f}s, expected < {_WINDOW + 1.0}s' # 5s task cannot have finished, so the envelope must be non-terminal. assert env.status not in ('completed', 'failed', 'cancelled'), env.status @@ -75,7 +75,7 @@ def test_poll_interval_satisfies_the_constant_chain(): def test_both_retrieve_endpoints_share_one_interval_declaration(): - """R2#6: one declaration point, and no endpoint reading os.environ on its own. + """One declaration point, and no endpoint reading os.environ on its own. Also pins the measured decision: a FIXED interval, not exponential backoff. The backoff variant was implemented, measured on real PPU hardware, and reverted -- @@ -99,7 +99,7 @@ def test_both_retrieve_endpoints_share_one_interval_declaration(): def test_gateway_guard_warns_once_per_value_not_once_per_request(monkeypatch): - """D5 guard must be audible but not spam. + """The poll-window guard must be audible but not spam. ``long_poll_window()`` runs on the hot path of both retrieve endpoints, not only at startup, so an unguarded warning would repeat on every retrieve request (~2/s during diff --git a/tests/server/lifecycle/test_tinker_retrieve_regression.py b/tests/server/lifecycle/test_tinker_retrieve_regression.py index 257c283a..4af4611f 100644 --- a/tests/server/lifecycle/test_tinker_retrieve_regression.py +++ b/tests/server/lifecycle/test_tinker_retrieve_regression.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Tinker /retrieve_future wire regression (T6.3 / Property 6 / R8#8). +"""Tinker /retrieve_future wire regression. The tinker endpoint's response shape and status-code semantics must be unchanged by this spec, across all three shapes: ``try_again`` / ``{error, category}`` / bare @@ -10,7 +10,7 @@ from fastapi import FastAPI from fastapi.testclient import TestClient -from twinkle.server.gateway.tinker_handlers import _register_tinker_routes +from twinkle.server.gateway.tinker_handlers import _register_gateway_tinker_routes class _State: @@ -30,7 +30,7 @@ def __init__(self, record): def _client(record): app = FastAPI() - _register_tinker_routes(app, lambda: _Gateway(record)) + _register_gateway_tinker_routes(app, lambda: _Gateway(record)) return TestClient(app) diff --git a/tests/server/model/test_tinker_compat_output.py b/tests/server/model/test_tinker_compat_output.py index 2041aefa..61c07557 100644 --- a/tests/server/model/test_tinker_compat_output.py +++ b/tests/server/model/test_tinker_compat_output.py @@ -1,7 +1,7 @@ import torch from tinker import types -from twinkle.server.common.datum import extract_rl_features_for_loss +from twinkle.server.model.tinker_datum import extract_rl_features_for_loss from twinkle.server.model.backends.common import TwinkleCompatModelBase diff --git a/tests/server/model/test_tinker_handlers.py b/tests/server/model/test_tinker_handlers.py index 52079198..7e78f306 100644 --- a/tests/server/model/test_tinker_handlers.py +++ b/tests/server/model/test_tinker_handlers.py @@ -4,7 +4,7 @@ from tinker import types from unittest.mock import AsyncMock, MagicMock, patch -from twinkle.server.model.tinker_handlers import _register_tinker_routes +from twinkle.server.model.tinker_handlers import _register_model_tinker_routes class _DummyManagement: @@ -29,7 +29,7 @@ def _datum(): async def test_tinker_dpo_forward_backward_requires_per_dp_pairs(): management = _DummyManagement() app = FastAPI() - _register_tinker_routes(app, lambda: management) + _register_model_tinker_routes(app, lambda: management) body = types.ForwardBackwardRequest( model_id='model1', @@ -91,7 +91,7 @@ async def test_save_weights_for_sampler_path_mode_returns_path(mock_create_ckpt_ management = _SaveWeightsDummyManagement() app = FastAPI() - _register_tinker_routes(app, lambda: management) + _register_model_tinker_routes(app, lambda: management) body = types.SaveWeightsForSamplerRequest( model_id='model1', @@ -119,7 +119,7 @@ async def test_save_weights_for_sampler_session_mode_returns_none_path(mock_crea management = _SaveWeightsDummyManagement() app = FastAPI() - _register_tinker_routes(app, lambda: management) + _register_model_tinker_routes(app, lambda: management) body = types.SaveWeightsForSamplerRequest( model_id='model1', diff --git a/tests/server/model/test_twinkle_async_inputs.py b/tests/server/model/test_twinkle_async_inputs.py index 106d1b04..3e98bf2b 100644 --- a/tests/server/model/test_twinkle_async_inputs.py +++ b/tests/server/model/test_twinkle_async_inputs.py @@ -6,7 +6,7 @@ import twinkle_client.types as types from twinkle.server.model.data_plane_inputs import model_result_rows -from twinkle.server.model.twinkle_handlers import _register_twinkle_routes +from twinkle.server.model.twinkle_handlers import _register_model_twinkle_routes def test_model_result_rows_keeps_one_output_row_per_sample() -> None: @@ -86,7 +86,7 @@ async def call_backend(self, fn, /, *args, admit=True, **kwargs): async def test_forward_backward_resolves_multiple_data_refs_and_field_kwargs() -> None: management = _SchedulingManagement() app = FastAPI() - _register_twinkle_routes(app, lambda: management) + _register_model_twinkle_routes(app, lambda: management) route = next( route for route in app.routes if getattr(route, 'path', None) == '/twinkle/forward_backward_from_data_plane') request = Request({'type': 'http', 'headers': []}) @@ -131,7 +131,7 @@ async def test_forward_backward_binds_nested_dpo_ref_logps_without_coercion() -> }, ] app = FastAPI() - _register_twinkle_routes(app, lambda: management) + _register_model_twinkle_routes(app, lambda: management) route = next( route for route in app.routes if getattr(route, 'path', None) == '/twinkle/forward_backward_from_data_plane') request = Request({'type': 'http', 'headers': []}) diff --git a/tests/server/sampler/test_twinkle_async_rows.py b/tests/server/sampler/test_twinkle_async_rows.py index b6a59df7..6c63ad63 100644 --- a/tests/server/sampler/test_twinkle_async_rows.py +++ b/tests/server/sampler/test_twinkle_async_rows.py @@ -69,7 +69,7 @@ def __init__(self): self.enabled = True self.scheduled = [] self.put_rows = None - self._task_queue_config = SimpleNamespace(effective_execution_timeout=60.0) + self.task_queue_config = SimpleNamespace(effective_execution_timeout=60.0) async def _on_request_start(self, _request): return 'token' diff --git a/tests/server/session_resource/test_contract.py b/tests/server/session_resource/test_contract.py index b60f70f4..477b509c 100644 --- a/tests/server/session_resource/test_contract.py +++ b/tests/server/session_resource/test_contract.py @@ -7,9 +7,9 @@ import pytest from twinkle.server.exceptions import RequestRejectedError -from twinkle.server.utils.session_resource.adapter import AdapterManagerMixin -from twinkle.server.utils.session_resource.base import SessionResourceMixin -from twinkle.server.utils.session_resource.processor import ProcessorManagerMixin +from twinkle.server.session_resource.adapter import AdapterManagerMixin +from twinkle.server.session_resource.base import SessionResourceMixin +from twinkle.server.session_resource.processor import ProcessorManagerMixin class _State: @@ -63,25 +63,25 @@ def test_registration_requires_session_id() -> None: async def test_liveness_failure_has_hard_upper_bound_and_recovery_refreshes() -> None: state = _State([RuntimeError('down'), 109.0, RuntimeError('down'), RuntimeError('down')]) manager = _ResourceManager(state, timeout=10.0) - with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=100.0): + with mock.patch('twinkle.server.session_resource.base.time.time', return_value=100.0): manager.register_resource('r1', 'token', 'session') record = manager.get_resource_info('r1') - with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=105.0): + with mock.patch('twinkle.server.session_resource.base.time.time', return_value=105.0): assert await manager._is_session_alive('session', 'r1', record) is True - with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=110.0): + with mock.patch('twinkle.server.session_resource.base.time.time', return_value=110.0): assert await manager._is_session_alive('session', 'r1', record) is True assert manager.get_resource_info('r1')['last_liveness_confirmed_at'] == 110.0 - with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=115.0): + with mock.patch('twinkle.server.session_resource.base.time.time', return_value=115.0): assert await manager._is_session_alive('session', 'r1', record) is True - with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=120.0): + with mock.patch('twinkle.server.session_resource.base.time.time', return_value=120.0): assert await manager._is_session_alive('session', 'r1', record) is False @pytest.mark.asyncio async def test_countdown_restart_preserves_confirmation_time() -> None: manager = _ResourceManager(_State([100.0])) - with mock.patch('twinkle.server.utils.session_resource.base.time.time', return_value=100.0): + with mock.patch('twinkle.server.session_resource.base.time.time', return_value=100.0): manager.register_resource('r1', 'token', 'session') confirmed_at = manager.get_resource_info('r1')['last_liveness_confirmed_at'] diff --git a/tests/server/state/test_error_payload.py b/tests/server/state/test_error_payload.py index aabc1e2a..cf37d641 100644 --- a/tests/server/state/test_error_payload.py +++ b/tests/server/state/test_error_payload.py @@ -5,7 +5,7 @@ import pytest from pydantic import ValidationError -from twinkle.server.utils.task_errors import task_error_payload +from twinkle.server.task_errors import task_error_payload from twinkle_client.types.errors import ErrorCategory, ErrorPayload @@ -39,7 +39,7 @@ def test_error_category_matches_tinker_wire_values(): def test_tinker_sdk_parses_six_field_like_two_field(): - """R8#5: tinker's RequestFailedResponse ignores extra fields, so a six-field + """Tinker's RequestFailedResponse ignores extra fields, so a six-field payload parses equal to a two-field one on the declared fields. tinker's RequestErrorCategory values are lowercase ('server'), so the payloads diff --git a/tests/server/state/test_future_lifecycle.py b/tests/server/state/test_future_lifecycle.py index 064bed11..38994bb1 100644 --- a/tests/server/state/test_future_lifecycle.py +++ b/tests/server/state/test_future_lifecycle.py @@ -1,8 +1,8 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """State-hygiene tests for FutureManager cleanup and the do-not-regress guard. -Spec: T5.6 / R9#5 / R9#6 / Property 6 / Property 7. Both shipped backends need -infrastructure (``memory`` starts a detached Ray actor, ``redis`` needs a server), +Both shipped backends need infrastructure (``memory`` starts a detached Ray actor, +``redis`` needs a server), so these pure ``FutureManager`` semantics run against the dict-backed fake below. """ from __future__ import annotations @@ -116,7 +116,7 @@ async def test_non_terminal_orphan_is_failed_not_deleted(manager): await _store(manager, 'r2', 'running', replica_id='dead-replica') await manager.cleanup_expired(cutoff_time=time.time() + 10, alive_replica_ids={'replica-A'}) rec = await manager.get('r2') - assert rec is not None # NOT deleted (Property 6) + assert rec is not None # NOT deleted assert rec.status == 'failed' assert rec.result is None assert rec.failure.reason_code == 'orphaned_replica' diff --git a/tests/server/state/test_leader_election.py b/tests/server/state/test_leader_election.py index f1f8955c..792989c3 100644 --- a/tests/server/state/test_leader_election.py +++ b/tests/server/state/test_leader_election.py @@ -14,7 +14,7 @@ from twinkle.server.state import ServerState from twinkle.server.state.backend.memory_backend import RayActorBackend -from twinkle.server.state.server_state import LEADER_KEY, LEASE_RENEW +from twinkle.server.state.cleanup_coordinator import LEADER_KEY, LEASE_RENEW from twinkle.server.telemetry import MetricsRegistry diff --git a/tests/server/static/backend_call_exemptions.py b/tests/server/static/backend_call_exemptions.py index f3705fa6..1d0307ff 100644 --- a/tests/server/static/backend_call_exemptions.py +++ b/tests/server/static/backend_call_exemptions.py @@ -4,7 +4,7 @@ This file is the SINGLE source of allowed Blocking_Backend_Call bypasses. It is consumed by this spec's check (``test_no_direct_backend_call.py``) and is intended to be consumed unchanged by the ``server-request-lifecycle`` spec's equivalent -check -- there must be exactly one physical copy, not one per spec (R2#8). +check -- there must be exactly one physical copy, not one per spec. Each entry is ``(module_relpath, function_name)`` where ``module_relpath`` is relative to ``src/twinkle/server`` and ``function_name`` is the innermost enclosing @@ -13,8 +13,8 @@ The allowed exemptions are: - the ray ``Queue.get`` inside ``sample_stream``'s ``_stream_queue``: it bridges the - sampler actor's process boundary and is bounded by the dedicated double-timeout of - R4#10-11 (T5.5), not by ``call_backend``; + sampler actor's process boundary and is bounded by a dedicated double-timeout, + not by ``call_backend``; - the ``.sample_stream_to_queue.remote(...)`` call inside ``sample_stream`` itself: streaming generation must keep producing while the HTTP response streams, so it cannot use ``call_backend`` as-is and carries its own double timeout. This is diff --git a/tests/server/static/test_no_degraded_path.py b/tests/server/static/test_no_degraded_path.py index 765f760a..dcac5283 100644 --- a/tests/server/static/test_no_degraded_path.py +++ b/tests/server/static/test_no_degraded_path.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Static check: no silent-degradation symbols remain (T7.8 / R6#6 / R9#4 / Property 9). +"""Static check: no silent-degradation symbols remain. One wildcard search covering eight symbols; each must occur zero times in its scope. The symbols are matched as identifiers (word boundaries) so that ``nccl_safe_megatron`` diff --git a/tests/server/static/test_no_direct_backend_call.py b/tests/server/static/test_no_direct_backend_call.py index 44449038..89573d37 100644 --- a/tests/server/static/test_no_direct_backend_call.py +++ b/tests/server/static/test_no_direct_backend_call.py @@ -1,8 +1,6 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Static check: no direct Blocking_Backend_Call in ``src/twinkle/server/**``. -Spec: T3.7 / R2#7 / R2#8 / Property 2. - Asserts that backend methods are invoked only through ``call_backend``. The check covers direct calls, aliases created with ``getattr``, and methods passed to generic thread executors. The scan range is the server directory rather than a file list. diff --git a/tests/server/static/test_utils_bucket_is_light.py b/tests/server/static/test_utils_bucket_is_light.py new file mode 100644 index 00000000..6e970622 --- /dev/null +++ b/tests/server/static/test_utils_bucket_is_light.py @@ -0,0 +1,34 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""The ``twinkle.server.utils`` bucket must re-export only the two +dependency-light helpers, not the queue / session-resource machinery. + +The original intent was to assert ``import twinkle.server.utils`` pulls no +OpenTelemetry. That side-effect is dominated by the parent package's eager +``twinkle.server.__init__`` -> ``launcher`` -> ``application_spec`` -> ``task_queue.config`` +chain (which triggers ``task_queue/__init__`` -> ``mixin`` -> telemetry) and is out of +this Requirement's scope, so we assert the directly-controlled property instead: the +bucket's re-export surface. Re-exporting the mixins is what used to make every one of the +five light callers drag in the OpenTelemetry SDK. +""" +import twinkle.server.utils as bucket + +# The two genuinely dependency-free helpers the five call sites actually use. +_LIGHT_EXPORTS = ('get_template_for_model', 'wrap_builder_with_device_group_env') +# The heavy machinery that must no longer be re-exported through the bucket. +_HEAVY_EXPORTS = ( + 'TaskQueueMixin', + 'TaskQueueConfig', + 'RateLimiter', + 'QueueState', + 'TaskStatus', + 'SessionResourceMixin', + 'AdapterManagerMixin', + 'ProcessorManagerMixin', +) + + +def test_bucket_exports_only_light_helpers(): + for name in _LIGHT_EXPORTS: + assert hasattr(bucket, name), f'{name} should be re-exported by the utils bucket' + leaked = [name for name in _HEAVY_EXPORTS if hasattr(bucket, name)] + assert not leaked, f'utils bucket must not re-export heavy machinery: {leaked}' diff --git a/tests/server/telemetry/test_metrics_cache_invalidation.py b/tests/server/telemetry/test_metrics_cache_invalidation.py new file mode 100644 index 00000000..4f1c0fdf --- /dev/null +++ b/tests/server/telemetry/test_metrics_cache_invalidation.py @@ -0,0 +1,20 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Per-deployment metric adapters must not survive ``MetricsRegistry.reset()``. + +``ensure_telemetry_initialized`` resets the registry *in order to* rebind instruments to a +real MeterProvider; an adapter cached at module level survived that and kept recording into +NoOp instruments for the life of the process. This test fails before (module-level +cache) and passes once the caches live on the registry instance. +""" +from twinkle.server.telemetry.metrics import MetricsRegistry, get_task_metrics + + +def test_task_metrics_rebind_after_registry_reset(): + first = get_task_metrics('Model') + MetricsRegistry.reset() + second = get_task_metrics('Model') + assert first is not second + # The point is the *instrument*, not the wrapper identity: assert the bound + # instrument object differs, so a cheap "return a new wrapper around the same + # instrument" implementation cannot pass. + assert first.execution_seconds._instrument is not second.execution_seconds._instrument diff --git a/tests/server/test_gateway_services.py b/tests/server/test_gateway_services.py index d4bab9ca..8950689a 100644 --- a/tests/server/test_gateway_services.py +++ b/tests/server/test_gateway_services.py @@ -5,7 +5,7 @@ import asyncio from pathlib import Path -from twinkle.server.gateway import services +from twinkle.server.gateway import use_cases as services class _State: @@ -59,8 +59,8 @@ async def _fixed_poll(_state, _request_id): app = FastAPI() monkeypatch.setattr(tinker_handlers, 'poll_future', _fixed_poll) monkeypatch.setattr(twinkle_handlers, 'poll_future', _fixed_poll) - tinker_handlers._register_tinker_routes(app, lambda: _FakeGateway()) - twinkle_handlers._register_twinkle_routes(app, lambda: _FakeGateway()) + tinker_handlers._register_gateway_tinker_routes(app, lambda: _FakeGateway()) + twinkle_handlers._register_gateway_twinkle_routes(app, lambda: _FakeGateway()) return TestClient(app) diff --git a/tests/server/test_runtime.py b/tests/server/test_runtime.py new file mode 100644 index 00000000..84e4837d --- /dev/null +++ b/tests/server/test_runtime.py @@ -0,0 +1,42 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Directed tests for ``server/runtime.init_twinkle_runtime``. + +The processor deployment layer has no test under ``tests/server/`` and reuses this +function for it, so this pins the parameter trap it introduces: ``ncpu_proc_per_node`` is +forwarded only when provided (processor), never when unset (model/sampler), and the +DeviceMesh is built the same way as before (``mesh_dim_names`` -> ``DeviceMesh(**)`` else +``DeviceMesh.from_sizes(**)``). +""" +from __future__ import annotations + +from unittest import mock + +from twinkle.server.runtime import init_twinkle_runtime + + +def test_model_sampler_path_does_not_forward_ncpu_proc_per_node(): + with mock.patch('twinkle.initialize') as init, mock.patch('twinkle.DeviceMesh') as mesh: + mesh.from_sizes.return_value = 'MESH' + result = init_twinkle_runtime(False, 2, device_group='DG', device_mesh_dict={'sizes': [2]}) + assert result == 'MESH' + kwargs = init.call_args.kwargs + assert 'ncpu_proc_per_node' not in kwargs + assert kwargs['nproc_per_node'] == 2 and kwargs['groups'] == ['DG'] + mesh.from_sizes.assert_called_once_with(sizes=[2]) + + +def test_processor_path_forwards_ncpu_proc_per_node_and_named_mesh(): + with mock.patch('twinkle.initialize') as init, mock.patch('twinkle.DeviceMesh') as mesh: + mesh.return_value = 'NAMED_MESH' + result = init_twinkle_runtime( + False, 1, device_group='DG', device_mesh_dict={'mesh_dim_names': ['dp']}, ncpu_proc_per_node=8) + assert result == 'NAMED_MESH' + assert init.call_args.kwargs['ncpu_proc_per_node'] == 8 + mesh.assert_called_once_with(mesh_dim_names=['dp']) + + +def test_mock_backend_returns_none_and_uses_single_cpu_proc(): + with mock.patch('twinkle.initialize') as init, mock.patch('twinkle.DeviceMesh'): + result = init_twinkle_runtime(True, 1, device_group='DG', device_mesh_dict={}) + assert result is None + assert init.call_args.kwargs['ncpu_proc_per_node'] == 1 diff --git a/tests/server/utils/task_queue/test_config.py b/tests/server/utils/task_queue/test_config.py index 878bfa12..60787f56 100644 --- a/tests/server/utils/task_queue/test_config.py +++ b/tests/server/utils/task_queue/test_config.py @@ -13,7 +13,7 @@ from hypothesis import strategies as st from pydantic import ValidationError -from twinkle.server.utils.task_queue.config import TaskQueueConfig +from twinkle.server.task_queue.config import TaskQueueConfig # ---------- defaults snapshot used by the default-value test -------------- # diff --git a/tests/server/utils/test_task_errors.py b/tests/server/utils/test_task_errors.py index 494dc535..d58f37f1 100644 --- a/tests/server/utils/test_task_errors.py +++ b/tests/server/utils/test_task_errors.py @@ -1,4 +1,4 @@ -from twinkle.server.utils.task_errors import task_error_payload +from twinkle.server.task_errors import task_error_payload from twinkle_client.types.errors import ErrorCategory diff --git a/tests/server/utils/test_task_queue_mixin.py b/tests/server/utils/test_task_queue_mixin.py index fd893cd7..24743f75 100644 --- a/tests/server/utils/test_task_queue_mixin.py +++ b/tests/server/utils/test_task_queue_mixin.py @@ -1,10 +1,10 @@ import asyncio import pytest -from twinkle.server.utils.task_queue.config import TaskQueueConfig -from twinkle.server.utils.task_queue.mixin import TaskQueueMixin -from twinkle.server.utils.task_queue.types import UserTaskError -from twinkle.server.utils.task_queue.worker import ComputeWorker +from twinkle.server.task_queue.config import TaskQueueConfig +from twinkle.server.task_queue.mixin import TaskQueueMixin +from twinkle.server.task_queue.types import UserTaskError +from twinkle.server.task_queue.worker import ComputeWorker class _DummyState: @@ -70,7 +70,7 @@ async def test_preflight_rejects_batch_without_per_dp_multiple(): batch_size_multiple=2, ) - # Property 3: a rejection writes no future record. + # A rejection writes no future record. assert queue.state.records == [] @@ -180,7 +180,7 @@ async def work(): finally: await queue._compute_worker.stop() - # Property 0: the failure payload rides the envelope's `error` field. + # The failure payload rides the envelope's `error` field. assert env.status == 'failed' assert env.error is not None and 'model failed' in env.error.error assert any(args[1] == 'failed' for args, _ in queue.state.records) diff --git a/tests/server/validation/test_request_wire.py b/tests/server/validation/test_request_wire.py index 0c346e32..c0665462 100644 --- a/tests/server/validation/test_request_wire.py +++ b/tests/server/validation/test_request_wire.py @@ -13,7 +13,9 @@ from fastapi.testclient import TestClient from pydantic import ValidationError -from twinkle.server.validation.errors import register_validation_error_handler +from fastapi.exceptions import RequestValidationError + +from twinkle.server.deployment import validation_error_handler from twinkle_client.types import model as model_types from twinkle_client.types.base import StrictRequest @@ -27,7 +29,7 @@ def client() -> TestClient: check did not happen. """ app = FastAPI() - register_validation_error_handler(app) + app.add_exception_handler(RequestValidationError, validation_error_handler) @app.post('/forward') async def forward(body: model_types.ForwardRequest): diff --git a/tests/twinkle_client/test_error_parsing.py b/tests/twinkle_client/test_error_parsing.py index 76b688a1..f55fd1ed 100644 --- a/tests/twinkle_client/test_error_parsing.py +++ b/tests/twinkle_client/test_error_parsing.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Client error-response parsing (T3.4 / Requirement 3 #7-#11).""" +"""Client error-response parsing ( / Requirement 3 #7-#11).""" from __future__ import annotations import pytest @@ -26,11 +26,11 @@ def json(self): def test_structured_error_reads_top_level_fields(): - """R3#7/#8: top-level category/error_code/request_id are preferred.""" + """Top-level category/error_code/request_id are preferred.""" resp = _Resp(422, body={'error': 'bad input', 'category': 'user', 'error_code': 422, 'request_id': 'req-7'}) with pytest.raises(TwinkleHTTPError) as exc: _handle_response(resp) - assert isinstance(exc.value, requests.HTTPError) # R3#8: existing except clauses keep working + assert isinstance(exc.value, requests.HTTPError) # existing except clauses keep working assert exc.value.status_code == 422 assert exc.value.error_code == 422 assert exc.value.category == 'user' @@ -96,7 +96,7 @@ def test_server_traceback_is_preserved(): def test_410_raises_stop_iteration_not_http_error(): - """R3#9: 410 keeps raising StopIteration, not an HTTP error.""" + """410 keeps raising StopIteration, not an HTTP error.""" resp = _Resp(410, body={'detail': 'exhausted'}) with pytest.raises(StopIteration): _handle_response(resp) diff --git a/tests/twinkle_client/test_future_layer.py b/tests/twinkle_client/test_future_layer.py index 9b389680..f4da3c0c 100644 --- a/tests/twinkle_client/test_future_layer.py +++ b/tests/twinkle_client/test_future_layer.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Client_Future_Layer unit tests (T2.2 / Requirement 4). +"""Client_Future_Layer unit tests ( / Requirement 4). ``resolve`` is exercised against fabricated envelopes and a monkeypatched ``_post_retrieve``; no server or network is involved. @@ -40,7 +40,7 @@ def _running(): def test_terminal_submit_issues_no_retrieve(monkeypatch): - """R8#1: a task terminal in the submit envelope makes zero retrieve calls.""" + """A task terminal in the submit envelope makes zero retrieve calls.""" def _boom(_request_id, _transport): raise AssertionError('retrieve must not be called for a terminal submit') @@ -51,7 +51,7 @@ def _boom(_request_id, _transport): def test_terminal_submit_failure_raises_taskfailed_with_payload(monkeypatch): - """Property 0: a failure in the submit envelope raises TaskFailedError, payload intact.""" + """A failure in the submit envelope raises TaskFailedError, payload intact.""" monkeypatch.setattr(_future, '_post_retrieve', lambda _r, _transport: pytest.fail('no retrieve')) with pytest.raises(TaskFailedError) as exc: _future.resolve(_failed(), model_cls=_Model) @@ -59,11 +59,11 @@ def test_terminal_submit_failure_raises_taskfailed_with_payload(monkeypatch): assert exc.value.category == 'server' assert exc.value.request_id == 'r' assert exc.value.error_code == 500 - assert not isinstance(exc.value, requests.HTTPError) # R3#10 + assert not isinstance(exc.value, requests.HTTPError) def test_model_cls_none_returns_none_result(monkeypatch): - """R4#9: a method that returned None before still returns None (not swallowed).""" + """A method that returned None before still returns None (not swallowed).""" monkeypatch.setattr(_future, '_post_retrieve', lambda _r, _transport: pytest.fail('no retrieve')) assert _future.resolve(_completed(None), model_cls=None) is None @@ -121,7 +121,7 @@ def test_total_timeout_raises_wait_timeout(monkeypatch): def test_success_resets_both_retry_counters(monkeypatch): - """R4#8: a successful reply zeroes both counters, so intermittent 404s never sum up.""" + """A successful reply zeroes both counters, so intermittent 404s never sum up.""" seq = [] def _mixed(_request_id, _transport): diff --git a/tests/twinkle_client/test_types_contract.py b/tests/twinkle_client/test_types_contract.py index 03b5dfa6..d5cfc9a7 100644 --- a/tests/twinkle_client/test_types_contract.py +++ b/tests/twinkle_client/test_types_contract.py @@ -1,8 +1,8 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Contract-base consistency and naming-disambiguation tests. -- T6.2 / R7#9: ``QueueStateLiteral`` value set equals the server ``QueueState`` enum. -- T6.3 / R7#7: naming disambiguation guard. +- ``QueueStateLiteral`` value set equals the server ``QueueState`` enum. +- naming disambiguation guard. The two SDKs already share public names. The contract freezes that legacy set and rejects new collisions while requiring explicit aliases when both SDKs are imported @@ -15,7 +15,7 @@ import typing import twinkle -from twinkle.server.utils.task_queue.types import QueueState +from twinkle.server.task_queue.types import QueueState from twinkle_client.types.errors import QueueStateLiteral from twinkle_client.types.server import GetServerCapabilitiesResponse From 15acc7fd9fe51227ad25caa865e8eab36831abb5 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 21 Sep 2026 02:59:03 +0000 Subject: [PATCH 26/32] style: satisfy pre-commit lint (isort import order, pyupgrade unquotes TYPE_CHECKING forward-refs, flake8 line length) --- src/twinkle/server/lifecycle/submit.py | 4 ++-- src/twinkle_agentic/async_rl/data_plane.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/twinkle/server/lifecycle/submit.py b/src/twinkle/server/lifecycle/submit.py index a69e3157..cc767fd5 100644 --- a/src/twinkle/server/lifecycle/submit.py +++ b/src/twinkle/server/lifecycle/submit.py @@ -97,7 +97,7 @@ def backend_kwargs(body: Any) -> dict[str, Any]: return kwargs -def input_metrics(self: 'DataParallelDeployment', body: Any, *, data_parallel: bool = False) -> dict[str, Any]: +def input_metrics(self: DataParallelDeployment, body: Any, *, data_parallel: bool = False) -> dict[str, Any]: """Seam C: scheduling metrics (input_tokens, and batch_size/data_world_size). Reads validated wire models, so no isinstance guards: ``inputs`` is a list and @@ -121,7 +121,7 @@ def resolve_twinkle_adapter_name(request: Request, adapter_name: str | None) -> async def run_submit( - self: 'QueuedDeployment', + self: QueuedDeployment, request: Request, body: Any, *, diff --git a/src/twinkle_agentic/async_rl/data_plane.py b/src/twinkle_agentic/async_rl/data_plane.py index 5bacc4b9..58423ced 100644 --- a/src/twinkle_agentic/async_rl/data_plane.py +++ b/src/twinkle_agentic/async_rl/data_plane.py @@ -5,8 +5,8 @@ from typing import Any, Sequence -from twinkle.data_format import REQUIRED_MODEL_INPUT_FIELDS, ROLLOUT_TRAIN_FIELDS, columns_to_tq_fields, rows_to_tq_fields - +from twinkle.data_format import (REQUIRED_MODEL_INPUT_FIELDS, ROLLOUT_TRAIN_FIELDS, columns_to_tq_fields, + rows_to_tq_fields) from .native_tq import (AsyncTQClient, append_fields, batch_size_for_groups, clear_partition, fetch_ready_batch, metadata_size, preallocate_partition, set_sample_tags, split_batch_meta) from .types import ClaimedBatch, LoraContext, PartitionAdmission, PreparedPartition, PromptGroup, RolloutOutput From b00d385b8c9630222aa5b03db82ae671b1026ce0 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 21 Sep 2026 03:10:01 +0000 Subject: [PATCH 27/32] style: pyupgrade unquotes forward-ref annotation in state/model_manager.py (pre-commit --all-files clean) --- src/twinkle/server/state/model_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/twinkle/server/state/model_manager.py b/src/twinkle/server/state/model_manager.py index ad286c10..26e7b4c0 100644 --- a/src/twinkle/server/state/model_manager.py +++ b/src/twinkle/server/state/model_manager.py @@ -38,7 +38,7 @@ def _counter_delta_transform(existing: object, *, delta: int) -> int: return new if new > 0 else 0 -async def _remove_with_record(manager: 'ModelManager', model_id: str, record: ModelRecord) -> bool: +async def _remove_with_record(manager: ModelManager, model_id: str, record: ModelRecord) -> bool: """Remove a known record without exposing it in the public method signature.""" removed = await BaseManager.remove(manager, model_id) if not removed: From b20608fad522026ff45fc1dfe4cfcb4be94c1fe1 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 21 Sep 2026 03:43:03 +0000 Subject: [PATCH 28/32] test(server): streamline error payload coverage --- tests/server/contract/update_baseline.py | 4 +++- tests/server/start_e2e_server.py | 4 +++- tests/server/state/test_error_payload.py | 12 ++++++++++ tests/server/utils/test_task_errors.py | 28 ------------------------ 4 files changed, 18 insertions(+), 30 deletions(-) delete mode 100644 tests/server/utils/test_task_errors.py diff --git a/tests/server/contract/update_baseline.py b/tests/server/contract/update_baseline.py index c819bff6..f5b47097 100644 --- a/tests/server/contract/update_baseline.py +++ b/tests/server/contract/update_baseline.py @@ -1,5 +1,7 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Regenerate the client-API contract snapshots. +"""Manual contract-maintenance helper; it is not a pytest test or CI entry point. + +Regenerate the client-API contract snapshots. Run with:: diff --git a/tests/server/start_e2e_server.py b/tests/server/start_e2e_server.py index 922283de..4af2e2b8 100644 --- a/tests/server/start_e2e_server.py +++ b/tests/server/start_e2e_server.py @@ -1,4 +1,6 @@ -"""One-click: restart Ray cluster + launch Twinkle server + wait until ready. +"""Manual PPU E2E helper; it is not a pytest test or CI entry point. + +One-click: restart Ray cluster + launch Twinkle server + wait until ready. Usage: python start_e2e_server.py # default config (transformers LoRA) diff --git a/tests/server/state/test_error_payload.py b/tests/server/state/test_error_payload.py index cf37d641..3b8ff7c1 100644 --- a/tests/server/state/test_error_payload.py +++ b/tests/server/state/test_error_payload.py @@ -23,6 +23,18 @@ def test_overlong_traceback_is_trimmed_tail_kept_with_marker(): assert tb.endswith('line\n') # tail preserved +def test_task_error_payload_shapes_and_sanitizes_errors(): + payload = task_error_payload( + 'RuntimeError: boom\n File "/server/path.py", line 1', request_id='req_1', error_code=500) + + assert payload == { + 'error': 'RuntimeError: boom', + 'category': ErrorCategory.Server.value, + 'error_code': 500, + 'request_id': 'req_1', + } + + def test_user_category_carries_no_traceback(): payload = task_error_payload( 'invalid field', request_id='req_2', error_code=422, diff --git a/tests/server/utils/test_task_errors.py b/tests/server/utils/test_task_errors.py deleted file mode 100644 index d58f37f1..00000000 --- a/tests/server/utils/test_task_errors.py +++ /dev/null @@ -1,28 +0,0 @@ -from twinkle.server.task_errors import task_error_payload -from twinkle_client.types.errors import ErrorCategory - - -def test_task_error_payload_builds_error_payload_dict(): - error = 'RuntimeError: No lora available for tenant session-default. Max loras: 3' - - payload = task_error_payload(error, request_id='req_1', error_code=500) - - assert payload['error'] == error - assert payload['category'] == ErrorCategory.Server.value - assert payload['error_code'] == 500 - assert payload['request_id'] == 'req_1' - assert 'traceback' not in payload - - -def test_task_error_payload_user_category_drops_traceback(): - payload = task_error_payload( - 'bad input', request_id='req_2', error_code=400, category=ErrorCategory.User, traceback_text='Traceback...') - - assert payload['category'] == ErrorCategory.User.value - assert 'traceback' not in payload - - -def test_error_summary_is_single_line(): - payload = task_error_payload( - 'RuntimeError: boom\n File "/server/path.py", line 1', request_id='req-lines') - assert payload['error'] == 'RuntimeError: boom' From d6d48fb7e0da68c3f33b97a0ec7e7fa5de72cda5 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 21 Sep 2026 09:46:50 +0000 Subject: [PATCH 29/32] refactor: harden wire contract and client transport --- README.md | 2 +- README_ZH.md | 2 +- .../async_rl/client_orchestrated_grpo.py | 2 +- cookbook/client/tinker/dpo.py | 3 +- cookbook/client/tinker/lora.py | 2 +- cookbook/client/tinker/multi_modal.py | 2 +- cookbook/client/tinker/sample.py | 2 +- cookbook/client/tinker/self_cognition.py | 2 +- cookbook/client/tinker/short_math_grpo.py | 2 +- cookbook/client/tinker/upload_to_hub.py | 3 +- cookbook/client/twinkle/dpo.py | 3 +- cookbook/client/twinkle/embedding.py | 3 +- cookbook/client/twinkle/multi_modal.py | 3 +- cookbook/client/twinkle/multi_turn_rollout.py | 3 +- cookbook/client/twinkle/sample.py | 3 +- cookbook/client/twinkle/self_cognition.py | 3 +- cookbook/client/twinkle/short_math_grpo.py | 3 +- cookbook/client/twinkle/upload_to_hub.py | 3 +- .../Usage Guide/Introduction-with-Qwen3.5.md | 2 +- .../Tinker-Compatible-Client.md | 6 +- .../Server and Client/Twinkle-Client.md | 2 +- .../Usage Guide/Train-as-a-Service.md | 2 +- ...00\344\275\263\345\256\236\350\267\265.md" | 2 +- ...71\345\256\242\346\210\267\347\253\257.md" | 6 +- ...le\345\256\242\346\210\267\347\253\257.md" | 2 +- ...55\347\273\203\346\234\215\345\212\241.md" | 2 +- notebook/dpo.ipynb | 6 +- notebook/multi_modal.ipynb | 4 +- notebook/sample.ipynb | 2 +- notebook/self_cognition.ipynb | 4 +- notebook/short_math_grpo.ipynb | 6 +- src/twinkle/__init__.py | 32 ++- src/twinkle/_lazy_module.py | 61 ++++++ src/twinkle/dataset/base.py | 3 +- src/twinkle/model/__init__.py | 2 +- src/twinkle/model/megatron/__init__.py | 2 +- src/twinkle/protocol/__init__.py | 22 ++ .../http => twinkle/protocol}/headers.py | 0 .../common => twinkle/protocol}/json_utils.py | 0 .../common => twinkle/protocol}/serialize.py | 40 +++- .../protocol}/types/__init__.py | 4 +- .../protocol}/types/base.py | 2 +- .../protocol}/types/checkpoint.py | 0 .../protocol}/types/component.py | 0 .../protocol}/types/data.py | 4 +- .../protocol}/types/errors.py | 0 .../protocol}/types/lifecycle.py | 2 +- .../protocol}/types/model.py | 89 +++------ .../protocol}/types/processor.py | 2 +- .../protocol}/types/sampler.py | 6 +- .../protocol}/types/server.py | 4 - .../protocol}/types/session.py | 0 .../protocol}/types/training.py | 0 .../server/checkpoint/checkpoint_manager.py | 2 +- src/twinkle/server/checkpoint/twinkle.py | 8 +- src/twinkle/server/config/server_config.py | 2 - src/twinkle/server/data_plane/handlers.py | 2 +- src/twinkle/server/data_plane/proxy.py | 4 +- src/twinkle/server/data_plane/store.py | 4 +- src/twinkle/server/deployment.py | 2 +- src/twinkle/server/exceptions.py | 2 +- src/twinkle/server/gateway/app.py | 2 +- src/twinkle/server/gateway/openai_handlers.py | 2 +- src/twinkle/server/gateway/proxy.py | 4 +- src/twinkle/server/gateway/tinker_handlers.py | 2 +- .../server/gateway/twinkle_handlers.py | 2 +- src/twinkle/server/lifecycle/envelope.py | 4 +- src/twinkle/server/lifecycle/submit.py | 10 +- src/twinkle/server/middleware/auth.py | 2 +- src/twinkle/server/model/data_plane_inputs.py | 2 +- src/twinkle/server/model/twinkle_handlers.py | 6 +- .../server/processor/twinkle_handlers.py | 5 +- .../server/sampler/twinkle_handlers.py | 8 +- src/twinkle/server/session_resource/base.py | 14 -- src/twinkle/server/state/__init__.py | 2 - src/twinkle/server/state/backend/base.py | 11 +- src/twinkle/server/state/config_manager.py | 88 -------- src/twinkle/server/state/model_manager.py | 6 - src/twinkle/server/state/replica_registry.py | 20 -- src/twinkle/server/state/server_state.py | 31 +-- src/twinkle/server/task_errors.py | 2 +- src/twinkle/server/task_queue/config.py | 4 +- src/twinkle/server/task_queue/mixin.py | 4 +- src/twinkle/server/task_queue/rate_limiter.py | 56 ++---- src/twinkle/server/task_queue/types.py | 1 - src/twinkle/server/task_queue/worker.py | 2 +- .../server/validation/backend_compat.py | 2 +- src/twinkle/utils/import_utils.py | 60 ------ src/twinkle_client/__init__.py | 18 +- src/twinkle_client/_future.py | 12 +- src/twinkle_client/_request_builder.py | 34 ++-- src/twinkle_client/common/__init__.py | 6 +- src/twinkle_client/common/component_rpc.py | 20 +- src/twinkle_client/common/remote_component.py | 45 +++++ src/twinkle_client/data_plane.py | 6 +- src/twinkle_client/dataloader/dataloader.py | 27 ++- src/twinkle_client/dataset/base.py | 12 +- .../dataset/iterable_dataset.py | 24 +-- .../dataset/iterable_packing_dataset.py | 17 +- src/twinkle_client/dataset/lazy_dataset.py | 42 +--- src/twinkle_client/dataset/packing_dataset.py | 7 +- src/twinkle_client/exceptions.py | 2 +- src/twinkle_client/http/__init__.py | 6 +- src/twinkle_client/http/client.py | 188 +++++++----------- src/twinkle_client/http/context.py | 13 ++ src/twinkle_client/manager.py | 64 +++--- .../model/multi_lora_transformers.py | 4 +- src/twinkle_client/processor/__init__.py | 3 - src/twinkle_client/processor/base.py | 35 ---- src/twinkle_client/sampler/vllm_sampler.py | 12 +- .../skills/bundled/twinkle-training.md | 12 +- src/twinkle_client/utils/patch_tinker.py | 2 +- .../contract/test_client_api_contract.py | 4 +- .../contract/test_protocol_migration.py | 35 ++++ tests/server/data_plane/test_proxy.py | 4 +- tests/server/data_plane/test_store.py | 4 +- tests/server/gateway/test_openai_handlers.py | 2 +- tests/server/gateway/test_proxy.py | 2 +- tests/server/integration/e2e_helpers.py | 4 +- .../server/integration/test_full_cycle_e2e.py | 3 +- .../server/integration/test_full_param_e2e.py | 3 +- .../lifecycle/test_envelope_coverage.py | 2 +- .../server/lifecycle/test_run_submit_dedup.py | 2 +- tests/server/lifecycle/test_static_guards.py | 8 +- .../server/lifecycle/test_submit_peek_e2e.py | 2 +- .../server/model/test_twinkle_async_inputs.py | 4 +- .../server/sampler/test_twinkle_async_rows.py | 4 +- tests/server/state/fake_backend.py | 3 + tests/server/state/test_error_payload.py | 2 +- tests/server/state/test_future_lifecycle.py | 3 + tests/server/state/test_managers.py | 7 +- tests/server/state/test_redis_integration.py | 70 +------ .../test_client_architecture_imports.py | 8 +- tests/server/test_gateway_services.py | 2 +- tests/server/utils/test_rate_limiter.py | 58 ++++++ tests/server/validation/test_preflight.py | 4 +- tests/server/validation/test_request_wire.py | 12 +- tests/server/validation/test_wire_schema.py | 2 +- tests/twinkle_client/test_async_components.py | 2 +- .../test_client_multi_turn_rollout.py | 4 +- .../test_client_orchestrated_grpo.py | 2 +- tests/twinkle_client/test_component_rpc.py | 56 ++++++ tests/twinkle_client/test_data_plane_async.py | 2 +- tests/twinkle_client/test_future_layer.py | 34 +++- tests/twinkle_client/test_import_surface.py | 59 ++++++ .../twinkle_client/test_remote_components.py | 75 +++++++ tests/twinkle_client/test_request_builder.py | 6 +- tests/twinkle_client/test_transport.py | 57 +++++- tests/twinkle_client/test_types_contract.py | 34 +++- 149 files changed, 1035 insertions(+), 906 deletions(-) create mode 100644 src/twinkle/_lazy_module.py create mode 100644 src/twinkle/protocol/__init__.py rename src/{twinkle_client/http => twinkle/protocol}/headers.py (100%) rename src/{twinkle_client/common => twinkle/protocol}/json_utils.py (100%) rename src/{twinkle_client/common => twinkle/protocol}/serialize.py (77%) rename src/{twinkle_client => twinkle/protocol}/types/__init__.py (97%) rename src/{twinkle_client => twinkle/protocol}/types/base.py (98%) rename src/{twinkle_client => twinkle/protocol}/types/checkpoint.py (100%) rename src/{twinkle_client => twinkle/protocol}/types/component.py (100%) rename src/{twinkle_client => twinkle/protocol}/types/data.py (98%) rename src/{twinkle_client => twinkle/protocol}/types/errors.py (100%) rename src/{twinkle_client => twinkle/protocol}/types/lifecycle.py (97%) rename src/{twinkle_client => twinkle/protocol}/types/model.py (90%) rename src/{twinkle_client => twinkle/protocol}/types/processor.py (97%) rename src/{twinkle_client => twinkle/protocol}/types/sampler.py (95%) rename src/{twinkle_client => twinkle/protocol}/types/server.py (96%) rename src/{twinkle_client => twinkle/protocol}/types/session.py (100%) rename src/{twinkle_client => twinkle/protocol}/types/training.py (100%) delete mode 100644 src/twinkle/server/state/config_manager.py create mode 100644 src/twinkle_client/common/remote_component.py delete mode 100644 src/twinkle_client/processor/__init__.py delete mode 100644 src/twinkle_client/processor/base.py create mode 100644 tests/server/contract/test_protocol_migration.py create mode 100644 tests/server/utils/test_rate_limiter.py create mode 100644 tests/twinkle_client/test_component_rpc.py create mode 100644 tests/twinkle_client/test_import_surface.py create mode 100644 tests/twinkle_client/test_remote_components.py diff --git a/README.md b/README.md index fc64af16..dc73830b 100644 --- a/README.md +++ b/README.md @@ -252,7 +252,7 @@ if __name__ == '__main__': import os from tqdm import tqdm from tinker import types -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor diff --git a/README_ZH.md b/README_ZH.md index d7b3d66f..8cb110bb 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -241,7 +241,7 @@ if __name__ == '__main__': import os from tqdm import tqdm from tinker import types -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor diff --git a/cookbook/client/async_rl/client_orchestrated_grpo.py b/cookbook/client/async_rl/client_orchestrated_grpo.py index 0f316256..99f010be 100644 --- a/cookbook/client/async_rl/client_orchestrated_grpo.py +++ b/cookbook/client/async_rl/client_orchestrated_grpo.py @@ -17,7 +17,7 @@ from twinkle.reward import GSM8KAccuracyReward from twinkle_client import DataPlaneClient, init_twinkle_client from twinkle_client.async_rl import Worker, WorkerPipeline -from twinkle_client.common.json_utils import json_safe +from twinkle.protocol.json_utils import json_safe from twinkle_client.model import MultiLoraTransformersModel from twinkle_client.sampler import vLLMSampler diff --git a/cookbook/client/tinker/dpo.py b/cookbook/client/tinker/dpo.py index 6091e808..16e2de50 100644 --- a/cookbook/client/tinker/dpo.py +++ b/cookbook/client/tinker/dpo.py @@ -23,7 +23,8 @@ import swanlab from tinker import types -from twinkle import init_tinker_client, get_logger +from twinkle import get_logger +from twinkle_client import init_tinker_client from twinkle.dataset import Dataset, DatasetMeta, LazyDataset from twinkle.dataloader import DataLoader from twinkle.preprocessor import EmojiDPOProcessor diff --git a/cookbook/client/tinker/lora.py b/cookbook/client/tinker/lora.py index 2b2bbe86..da1a7abf 100644 --- a/cookbook/client/tinker/lora.py +++ b/cookbook/client/tinker/lora.py @@ -13,7 +13,7 @@ import os # Step 2: Initialize Tinker client before importing ServiceClient -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client init_tinker_client() diff --git a/cookbook/client/tinker/multi_modal.py b/cookbook/client/tinker/multi_modal.py index dae6a4dd..d26c5237 100644 --- a/cookbook/client/tinker/multi_modal.py +++ b/cookbook/client/tinker/multi_modal.py @@ -17,7 +17,7 @@ # ============================================================================= # Step 1: Initialize Tinker client (MUST be done before importing ServiceClient) # ============================================================================= -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client init_tinker_client() diff --git a/cookbook/client/tinker/sample.py b/cookbook/client/tinker/sample.py index 2d94c95a..69380388 100644 --- a/cookbook/client/tinker/sample.py +++ b/cookbook/client/tinker/sample.py @@ -9,7 +9,7 @@ from twinkle.data_format import Message, Trajectory from twinkle.template import Template, Qwen3_5Template -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client # Step 1: Initialize Tinker client init_tinker_client() diff --git a/cookbook/client/tinker/self_cognition.py b/cookbook/client/tinker/self_cognition.py index 200f7f13..a785087e 100644 --- a/cookbook/client/tinker/self_cognition.py +++ b/cookbook/client/tinker/self_cognition.py @@ -10,7 +10,7 @@ import numpy as np from tqdm import tqdm from tinker import types -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client from twinkle.data_format import Message, Trajectory from twinkle.template import Template from twinkle.dataloader import DataLoader diff --git a/cookbook/client/tinker/short_math_grpo.py b/cookbook/client/tinker/short_math_grpo.py index 10eabf41..168d759e 100644 --- a/cookbook/client/tinker/short_math_grpo.py +++ b/cookbook/client/tinker/short_math_grpo.py @@ -24,7 +24,7 @@ from tinker import types from typing import List, Tuple, Dict, Any -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client from twinkle import get_logger from twinkle.advantage import GRPOAdvantage from twinkle.dataloader import DataLoader diff --git a/cookbook/client/tinker/upload_to_hub.py b/cookbook/client/tinker/upload_to_hub.py index b527ca3c..32b860c9 100644 --- a/cookbook/client/tinker/upload_to_hub.py +++ b/cookbook/client/tinker/upload_to_hub.py @@ -22,7 +22,8 @@ import os -from twinkle import get_logger, init_twinkle_client +from twinkle import get_logger +from twinkle_client import init_twinkle_client from twinkle_client.model import MultiLoraTransformersModel logger = get_logger() diff --git a/cookbook/client/twinkle/dpo.py b/cookbook/client/twinkle/dpo.py index 8fe64de2..4b76e59d 100644 --- a/cookbook/client/twinkle/dpo.py +++ b/cookbook/client/twinkle/dpo.py @@ -12,7 +12,8 @@ from peft import LoraConfig from typing import Any, Dict, List -from twinkle import get_logger, init_twinkle_client +from twinkle import get_logger +from twinkle_client import init_twinkle_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import EmojiDPOProcessor diff --git a/cookbook/client/twinkle/embedding.py b/cookbook/client/twinkle/embedding.py index d020e8a8..bc1211d0 100644 --- a/cookbook/client/twinkle/embedding.py +++ b/cookbook/client/twinkle/embedding.py @@ -28,7 +28,8 @@ from peft import LoraConfig from typing import Any, Dict, List -from twinkle import get_logger, init_twinkle_client +from twinkle import get_logger +from twinkle_client import init_twinkle_client from twinkle.template import Qwen3_5Template dotenv.load_dotenv('.env') diff --git a/cookbook/client/twinkle/multi_modal.py b/cookbook/client/twinkle/multi_modal.py index 55e71ecf..20354f23 100644 --- a/cookbook/client/twinkle/multi_modal.py +++ b/cookbook/client/twinkle/multi_modal.py @@ -11,7 +11,8 @@ import torch from peft import LoraConfig -from twinkle import get_logger, init_twinkle_client +from twinkle import get_logger +from twinkle_client import init_twinkle_client from twinkle.data_format import Message, Trajectory from twinkle.dataloader import DataLoader from twinkle.dataset import DatasetMeta, LazyDataset diff --git a/cookbook/client/twinkle/multi_turn_rollout.py b/cookbook/client/twinkle/multi_turn_rollout.py index 6325976b..ba160286 100644 --- a/cookbook/client/twinkle/multi_turn_rollout.py +++ b/cookbook/client/twinkle/multi_turn_rollout.py @@ -26,7 +26,8 @@ from peft import LoraConfig from typing import Any, Dict, List, Tuple -from twinkle import get_logger, init_twinkle_client +from twinkle import get_logger +from twinkle_client import init_twinkle_client from twinkle.advantage import GRPOAdvantage from twinkle.data_format import SamplingParams from twinkle.template import Qwen3_5Template diff --git a/cookbook/client/twinkle/sample.py b/cookbook/client/twinkle/sample.py index be6471de..8449b4e5 100644 --- a/cookbook/client/twinkle/sample.py +++ b/cookbook/client/twinkle/sample.py @@ -13,7 +13,8 @@ import os from transformers import AutoTokenizer -from twinkle import get_logger, init_twinkle_client +from twinkle import get_logger +from twinkle_client import init_twinkle_client dotenv.load_dotenv('.env') logger = get_logger() diff --git a/cookbook/client/twinkle/self_cognition.py b/cookbook/client/twinkle/self_cognition.py index 3d304575..14547520 100644 --- a/cookbook/client/twinkle/self_cognition.py +++ b/cookbook/client/twinkle/self_cognition.py @@ -9,7 +9,8 @@ import os from peft import LoraConfig -from twinkle import get_logger, init_twinkle_client +from twinkle import get_logger +from twinkle_client import init_twinkle_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta diff --git a/cookbook/client/twinkle/short_math_grpo.py b/cookbook/client/twinkle/short_math_grpo.py index d58e5e9c..7ef88036 100644 --- a/cookbook/client/twinkle/short_math_grpo.py +++ b/cookbook/client/twinkle/short_math_grpo.py @@ -27,7 +27,8 @@ from peft import LoraConfig from typing import Any, Dict, List, Tuple -from twinkle import get_logger, init_twinkle_client +from twinkle import get_logger +from twinkle_client import init_twinkle_client from twinkle.advantage import GRPOAdvantage from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta diff --git a/cookbook/client/twinkle/upload_to_hub.py b/cookbook/client/twinkle/upload_to_hub.py index b884fb73..5dc510e2 100644 --- a/cookbook/client/twinkle/upload_to_hub.py +++ b/cookbook/client/twinkle/upload_to_hub.py @@ -19,7 +19,8 @@ import dotenv import os -from twinkle import get_logger, init_twinkle_client +from twinkle import get_logger +from twinkle_client import init_twinkle_client dotenv.load_dotenv('.env') logger = get_logger() diff --git a/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md b/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md index 22dc07df..ed8825ce 100644 --- a/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md +++ b/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md @@ -454,7 +454,7 @@ import os from tinker import types from tqdm import tqdm -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor diff --git a/docs/source_en/Usage Guide/Server and Client/Tinker-Compatible-Client.md b/docs/source_en/Usage Guide/Server and Client/Tinker-Compatible-Client.md index a530174e..37860646 100644 --- a/docs/source_en/Usage Guide/Server and Client/Tinker-Compatible-Client.md +++ b/docs/source_en/Usage Guide/Server and Client/Tinker-Compatible-Client.md @@ -6,7 +6,7 @@ The Tinker Client is suitable for scenarios with existing Tinker training code. ```python # Initialize Tinker client before importing ServiceClient -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client init_tinker_client() @@ -41,7 +41,7 @@ import os import numpy as np from tqdm import tqdm from tinker import types -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor @@ -132,7 +132,7 @@ You can also load saved checkpoints for inference: ```python import os from tinker import types -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client from twinkle.data_format import Message, Trajectory from twinkle.template import Template diff --git a/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md b/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md index 27fd1337..a41f5af2 100644 --- a/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md +++ b/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md @@ -222,7 +222,7 @@ The examples above are all single-turn training. If you want to do **multi-turn ```python from peft import LoraConfig -from twinkle import init_twinkle_client +from twinkle_client import init_twinkle_client from twinkle.advantage import GRPOAdvantage from twinkle.data_format import SamplingParams from twinkle.template import Qwen3_5Template diff --git a/docs/source_en/Usage Guide/Train-as-a-Service.md b/docs/source_en/Usage Guide/Train-as-a-Service.md index 286b5200..e1ae3913 100644 --- a/docs/source_en/Usage Guide/Train-as-a-Service.md +++ b/docs/source_en/Usage Guide/Train-as-a-Service.md @@ -72,7 +72,7 @@ from tinker import types from twinkle.data_format import Message, Trajectory from twinkle.template import Template -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client # Step 1: Initialize Tinker client init_tinker_client() diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" index c80b1e15..2130e742 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" @@ -454,7 +454,7 @@ import os from tinker import types from tqdm import tqdm -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Tinker\345\205\274\345\256\271\345\256\242\346\210\267\347\253\257.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Tinker\345\205\274\345\256\271\345\256\242\346\210\267\347\253\257.md" index a1f7e064..885bf59a 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Tinker\345\205\274\345\256\271\345\256\242\346\210\267\347\253\257.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Tinker\345\205\274\345\256\271\345\256\242\346\210\267\347\253\257.md" @@ -6,7 +6,7 @@ Tinker Client 适用于已有 Tinker 训练代码的场景。通过 `init_tinker ```python # 在导入 ServiceClient 之前，先初始化 Tinker 客户端 -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client init_tinker_client() @@ -41,7 +41,7 @@ import os import numpy as np from tqdm import tqdm from tinker import types -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor @@ -132,7 +132,7 @@ for i, seq in enumerate(result.sequences): ```python import os from tinker import types -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client from twinkle.data_format import Message, Trajectory from twinkle.template import Template diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" index 4a62e3ab..ad9dd952 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" @@ -221,7 +221,7 @@ model.set_lr_scheduler('default', lr_decay_steps=1000, max_lr=1e-4) ```python from peft import LoraConfig -from twinkle import init_twinkle_client +from twinkle_client import init_twinkle_client from twinkle.advantage import GRPOAdvantage from twinkle.data_format import SamplingParams from twinkle.template import Qwen3_5Template diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" index c57cf066..f7a77db0 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" @@ -73,7 +73,7 @@ from tinker import types from twinkle.data_format import Message, Trajectory from twinkle.template import Template -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client # Step 1: Initialize Tinker client init_tinker_client() diff --git a/notebook/dpo.ipynb b/notebook/dpo.ipynb index d2a13cfa..db63ad44 100644 --- a/notebook/dpo.ipynb +++ b/notebook/dpo.ipynb @@ -140,7 +140,7 @@ "\n", "from tinker import types\n", "from getpass import getpass\n", - "from twinkle import init_tinker_client, get_logger\n", + "from twinkle import get_logger\nfrom twinkle_client import init_tinker_client\n", "from twinkle.dataset import Dataset, DatasetMeta, LazyDataset\n", "from twinkle.dataloader import DataLoader\n", "from twinkle.preprocessor import EmojiDPOProcessor\n", @@ -433,7 +433,7 @@ "metadata": {}, "outputs": [], "source": [ - "from twinkle import init_twinkle_client\n", + "from twinkle_client import init_twinkle_client\n", "from twinkle_client.model import MultiLoraTransformersModel\n", "# 步骤 1：初始化 Twinkle 客户端。\n", "# Tinker 检查点（twinkle:// 路径）由同一检查点服务解析\n", @@ -478,7 +478,7 @@ "source": [ "# 推理示例（使用线上服务，无需本地 GPU）\n", "from tinker import types\n", - "from twinkle import init_tinker_client, get_logger\n", + "from twinkle import get_logger\nfrom twinkle_client import init_tinker_client\n", "from twinkle.data_format import Message, Trajectory\n", "from twinkle.template import Template\n", "\n", diff --git a/notebook/multi_modal.ipynb b/notebook/multi_modal.ipynb index 9bb52467..2715192a 100644 --- a/notebook/multi_modal.ipynb +++ b/notebook/multi_modal.ipynb @@ -360,7 +360,7 @@ "metadata": {}, "outputs": [], "source": [ - "from twinkle import init_twinkle_client\n", + "from twinkle_client import init_twinkle_client\n", "from twinkle_client.model import MultiLoraTransformersModel\n", "# 步骤 1：初始化 Twinkle 客户端。\n", "# Tinker 检查点（twinkle:// 路径）由同一检查点服务解析\n", @@ -407,7 +407,7 @@ "source": [ "# 推理示例（使用线上服务，无需本地 GPU）\n", "from tinker import types\n", - "from twinkle import init_tinker_client, get_logger\n", + "from twinkle import get_logger\nfrom twinkle_client import init_tinker_client\n", "from twinkle.data_format import Message, Trajectory\n", "from twinkle.template import Qwen3_5Template\n", "\n", diff --git a/notebook/sample.ipynb b/notebook/sample.ipynb index 5c3c18b1..7a6b641d 100644 --- a/notebook/sample.ipynb +++ b/notebook/sample.ipynb @@ -102,7 +102,7 @@ "source": [ "from tinker import types\n", "from getpass import getpass\n", - "from twinkle import init_tinker_client, get_logger\n", + "from twinkle import get_logger\nfrom twinkle_client import init_tinker_client\n", "from twinkle.data_format import Message, Trajectory\n", "from twinkle.template import Template\n", "\n", diff --git a/notebook/self_cognition.ipynb b/notebook/self_cognition.ipynb index 5e3383f1..61c0cd2c 100644 --- a/notebook/self_cognition.ipynb +++ b/notebook/self_cognition.ipynb @@ -118,7 +118,7 @@ "import os\n", "from tqdm import tqdm\n", "from tinker import types\n", - "from twinkle import init_tinker_client\n", + "from twinkle_client import init_tinker_client\n", "from twinkle.data_format import Message, Trajectory\n", "from twinkle.template import Template\n", "from twinkle.dataloader import DataLoader\n", @@ -386,7 +386,7 @@ "metadata": {}, "outputs": [], "source": [ - "from twinkle import init_twinkle_client\n", + "from twinkle_client import init_twinkle_client\n", "from twinkle_client.model import MultiLoraTransformersModel\n", "# 步骤 1：初始化 Twinkle 客户端。\n", "# Tinker 检查点（twinkle:// 路径）由同一检查点服务解析\n", diff --git a/notebook/short_math_grpo.ipynb b/notebook/short_math_grpo.ipynb index a0576678..fbe8ffc3 100644 --- a/notebook/short_math_grpo.ipynb +++ b/notebook/short_math_grpo.ipynb @@ -136,7 +136,7 @@ "from typing import List, Tuple, Dict, Any\n", "\n", "from getpass import getpass\n", - "from twinkle import get_logger, init_twinkle_client\n", + "from twinkle import get_logger\nfrom twinkle_client import init_twinkle_client\n", "from twinkle.reward.base import Reward\n", "from twinkle.advantage import GRPOAdvantage\n", "from twinkle.dataset import DatasetMeta, Dataset\n", @@ -528,7 +528,7 @@ "metadata": {}, "outputs": [], "source": [ - "from twinkle import init_twinkle_client\n", + "from twinkle_client import init_twinkle_client\n", "from twinkle_client.model import MultiLoraTransformersModel\n", "# 步骤 1：初始化 Twinkle 客户端。\n", "# Tinker 检查点（twinkle:// 路径）由同一检查点服务解析\n", @@ -573,7 +573,7 @@ "source": [ "# 推理示例（使用线上服务，无需本地 GPU）\n", "from tinker import types\n", - "from twinkle import init_tinker_client, get_logger\n", + "from twinkle import get_logger\nfrom twinkle_client import init_tinker_client\n", "from twinkle.data_format import Message, Trajectory\n", "from twinkle.template import Template\n", "\n", diff --git a/src/twinkle/__init__.py b/src/twinkle/__init__.py index f64917a5..f80a4d74 100644 --- a/src/twinkle/__init__.py +++ b/src/twinkle/__init__.py @@ -1,10 +1,32 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any + +from ._lazy_module import _LazyModule # noqa + + +def init_tinker_client(**kwargs) -> None: + """Compatibility entry point; prefer ``twinkle_client.init_tinker_client``.""" + from twinkle_client import init_tinker_client as _init_tinker_client + return _init_tinker_client(**kwargs) + + +def init_twinkle_client( + base_url: str | None = None, + api_key: str | None = None, + session_heartbeat_interval: int = 10, + **kwargs, +) -> Any: + """Compatibility entry point; prefer ``twinkle_client.init_twinkle_client``.""" + from twinkle_client import init_twinkle_client as _init_twinkle_client + return _init_twinkle_client( + base_url=base_url, + api_key=api_key, + session_heartbeat_interval=session_heartbeat_interval, + **kwargs, + ) -from .utils.import_utils import _LazyModule # noqa if TYPE_CHECKING: - from twinkle_client import init_tinker_client, init_twinkle_client from .infra import get_device_placement, initialize, is_master, remote_class, remote_function from .utils import (GPU, NPU, DeviceGroup, DeviceMesh, Platform, Plugin, check_unsafe, exists, find_free_port, find_node_ip, framework_util, get_logger, requires, torch_util, trust_remote_code) @@ -21,8 +43,6 @@ import sys - from twinkle_client import init_tinker_client, init_twinkle_client - sys.modules[__name__] = _LazyModule( __name__, globals()['__file__'], @@ -30,6 +50,6 @@ module_spec=__spec__, # noqa extra_objects={ 'init_tinker_client': init_tinker_client, - 'init_twinkle_client': init_twinkle_client + 'init_twinkle_client': init_twinkle_client, }, ) diff --git a/src/twinkle/_lazy_module.py b/src/twinkle/_lazy_module.py new file mode 100644 index 00000000..1a7c2d26 --- /dev/null +++ b/src/twinkle/_lazy_module.py @@ -0,0 +1,61 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Dependency-light lazy module implementation used by package initializers.""" +import importlib +import os +from itertools import chain +from types import ModuleType +from typing import Any + + +class _LazyModule(ModuleType): + """ + Module class that surfaces all objects but only performs associated imports when the objects are requested. + """ + + # Very heavily inspired by optuna.integration._IntegrationModule + # https://github.com/optuna/optuna/blob/master/optuna/integration/__init__.py + def __init__(self, name, module_file, import_structure, module_spec=None, extra_objects=None): + super().__init__(name) + self._modules = set(import_structure.keys()) + self._class_to_module = {} + for key, values in import_structure.items(): + for value in values: + self._class_to_module[value] = key + # Needed for autocompletion in an IDE + self.__all__ = list(import_structure.keys()) + list(chain(*import_structure.values())) + self.__file__ = module_file + self.__spec__ = module_spec + self.__path__ = [os.path.dirname(module_file)] + self._objects = {} if extra_objects is None else extra_objects + self._name = name + self._import_structure = import_structure + + # Needed for autocompletion in an IDE + def __dir__(self): + result = super().__dir__() + # The elements of self.__all__ that are submodules may or may not be in the dir already, depending on whether + # they have been accessed or not. So we only add the elements of self.__all__ that are not already in the dir. + for attr in self.__all__: + if attr not in result: + result.append(attr) + return result + + def __getattr__(self, name: str) -> Any: + if name in self._objects: + return self._objects[name] + if name in self._modules: + value = self._get_module(name) + elif name in self._class_to_module.keys(): + module = self._get_module(self._class_to_module[name]) + value = getattr(module, name) + else: + raise AttributeError(f'module {self.__name__} has no attribute {name}') + + setattr(self, name, value) + return value + + def _get_module(self, module_name: str): + return importlib.import_module('.' + module_name, self.__name__) + + def __reduce__(self): + return self.__class__, (self._name, self.__file__, self._import_structure) diff --git a/src/twinkle/dataset/base.py b/src/twinkle/dataset/base.py index c0fceb52..b064df27 100644 --- a/src/twinkle/dataset/base.py +++ b/src/twinkle/dataset/base.py @@ -11,8 +11,6 @@ from typing import Any, Callable, Dict, List, Optional, Type, Union import twinkle -from twinkle import preprocessor -from twinkle.hub import HubOperation from twinkle.infra import remote_class, remote_function from twinkle.preprocessor import DataFilter, Preprocessor from twinkle.template import Template @@ -197,6 +195,7 @@ def _load_dataset(dataset_meta: DatasetMeta, **kwargs): kwargs['na_filter'] = False dataset = load_dataset(file_type, **load_kwargs, **kwargs) else: + from twinkle.hub import HubOperation dataset = HubOperation.load_dataset(dataset_id, subset_name, split, **kwargs) # fix: Some dataset sources return DatasetDict instead of Dataset, which breaks downstream select/map calls. diff --git a/src/twinkle/model/__init__.py b/src/twinkle/model/__init__.py index 2b367bbd..02f9eefd 100644 --- a/src/twinkle/model/__init__.py +++ b/src/twinkle/model/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from typing import TYPE_CHECKING -from twinkle.utils.import_utils import _LazyModule +from twinkle._lazy_module import _LazyModule if TYPE_CHECKING: from .base import TwinkleModel diff --git a/src/twinkle/model/megatron/__init__.py b/src/twinkle/model/megatron/__init__.py index 0f462566..e7e611d2 100644 --- a/src/twinkle/model/megatron/__init__.py +++ b/src/twinkle/model/megatron/__init__.py @@ -6,7 +6,7 @@ # Follow the same LazyModule approach as `twinkle.model`: only import when those symbols are actually accessed. from typing import TYPE_CHECKING -from twinkle.utils.import_utils import _LazyModule +from twinkle._lazy_module import _LazyModule if TYPE_CHECKING: from .megatron import MegatronModel, MegatronStrategy diff --git a/src/twinkle/protocol/__init__.py b/src/twinkle/protocol/__init__.py new file mode 100644 index 00000000..9b1c3b8c --- /dev/null +++ b/src/twinkle/protocol/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Wire contract shared by the Twinkle server and client.""" +from .headers import (H_AUTH, H_AUTH_TWINKLE, H_MULTIPLEX, H_MULTIPLEX_LEGACY, H_REQUEST_ID, H_REQUEST_ID_LEGACY, + build_routing_headers) +from .json_utils import json_safe +from .serialize import deserialize_object, serialize_object +from .types import * # noqa: F403 +from .types import __all__ as _TYPES_ALL + +__all__ = [ + *_TYPES_ALL, + 'H_AUTH', + 'H_AUTH_TWINKLE', + 'H_MULTIPLEX', + 'H_MULTIPLEX_LEGACY', + 'H_REQUEST_ID', + 'H_REQUEST_ID_LEGACY', + 'build_routing_headers', + 'json_safe', + 'serialize_object', + 'deserialize_object', +] diff --git a/src/twinkle_client/http/headers.py b/src/twinkle/protocol/headers.py similarity index 100% rename from src/twinkle_client/http/headers.py rename to src/twinkle/protocol/headers.py diff --git a/src/twinkle_client/common/json_utils.py b/src/twinkle/protocol/json_utils.py similarity index 100% rename from src/twinkle_client/common/json_utils.py rename to src/twinkle/protocol/json_utils.py diff --git a/src/twinkle_client/common/serialize.py b/src/twinkle/protocol/serialize.py similarity index 77% rename from src/twinkle_client/common/serialize.py rename to src/twinkle/protocol/serialize.py index 41a56ed7..227b0b88 100644 --- a/src/twinkle_client/common/serialize.py +++ b/src/twinkle/protocol/serialize.py @@ -1,17 +1,36 @@ # Copyright (c) ModelScope Contributors. All rights reserved. +"""Tagged serialization for non-JSON domain objects used by the wire contract. + +Heavy domain types are resolved lazily so importing :mod:`twinkle.protocol` does +not load PEFT or the dataset implementation. +""" import json from dataclasses import fields +from functools import lru_cache from numbers import Number -from peft import LoraConfig from pydantic import BaseModel from typing import Any, Mapping -from twinkle.dataset import DatasetMeta - primitive_types = (str, Number, bool, type(None)) container_types = (Mapping, list, tuple, set, frozenset) basic_types = (*primitive_types, *container_types) -_DATASET_META_FIELDS = {field.name for field in fields(DatasetMeta)} + + +@lru_cache(maxsize=1) +def _dataset_meta_cls(): + from twinkle.dataset import DatasetMeta + return DatasetMeta + + +@lru_cache(maxsize=1) +def _dataset_meta_fields() -> frozenset[str]: + return frozenset(field.name for field in fields(_dataset_meta_cls())) + + +@lru_cache(maxsize=1) +def _lora_config_cls(): + from peft import LoraConfig + return LoraConfig def _serialize_data_slice(data_slice): @@ -43,12 +62,12 @@ def _deserialize_data_slice(data_slice): def serialize_object(obj) -> Any: if isinstance(obj, (bytes, bytearray, memoryview)): raise TypeError(f'Unsupported binary object: {type(obj).__name__}') - if isinstance(obj, DatasetMeta): - data = {name: getattr(obj, name) for name in _DATASET_META_FIELDS} + if isinstance(obj, _dataset_meta_cls()): + data = {name: getattr(obj, name) for name in _dataset_meta_fields()} data['data_slice'] = _serialize_data_slice(data.get('data_slice')) data['_TWINKLE_TYPE_'] = 'DatasetMeta' return json.dumps(data, ensure_ascii=False) - elif isinstance(obj, LoraConfig): + elif isinstance(obj, _lora_config_cls()): filtered_dict = {} for _subkey, _subvalue in obj.__dict__.items(): if isinstance(_subvalue, basic_types) and not _subkey.startswith('_'): @@ -79,11 +98,12 @@ def deserialize_object(data: str) -> Any: if '_TWINKLE_TYPE_' in data: _type = data.pop('_TWINKLE_TYPE_') if _type == 'DatasetMeta': - data = {key: value for key, value in data.items() if key in _DATASET_META_FIELDS} + fields_set = _dataset_meta_fields() + data = {key: value for key, value in data.items() if key in fields_set} data['data_slice'] = _deserialize_data_slice(data.get('data_slice')) - return DatasetMeta(**data) + return _dataset_meta_cls()(**data) elif _type == 'LoraConfig': - return LoraConfig(**data) + return _lora_config_cls()(**data) else: raise ValueError(f'Unsupported type: {_type}') else: diff --git a/src/twinkle_client/types/__init__.py b/src/twinkle/protocol/types/__init__.py similarity index 97% rename from src/twinkle_client/types/__init__.py rename to src/twinkle/protocol/types/__init__.py index bbc05e9d..01b27257 100644 --- a/src/twinkle_client/types/__init__.py +++ b/src/twinkle/protocol/types/__init__.py @@ -25,8 +25,7 @@ SampleRequest, SampleResponseModel, SampleResponseModelList, SamplerSetTemplateRequest, SamplerSetTemplateResponse) from .server import (CapacityInfoResponse, CheckpointPathResponse, ClientFeatures, DeleteCheckpointResponse, - ErrorResponse, GetServerCapabilitiesResponse, HealthResponse, ProtocolLimits, SupportedModel, - WeightsInfoRequest) + GetServerCapabilitiesResponse, HealthResponse, ProtocolLimits, SupportedModel, WeightsInfoRequest) from .session import CreateSessionRequest, CreateSessionResponse, SessionHeartbeatRequest, SessionHeartbeatResponse from .training import (Checkpoint, CheckpointsListResponse, CreateModelRequest, Cursor, LoraConfig, ParsedCheckpointTwinklePath, TrainingRun, TrainingRunsResponse, WeightsInfoResponse) @@ -136,7 +135,6 @@ 'CheckpointPathResponse', 'ClientFeatures', 'DeleteCheckpointResponse', - 'ErrorResponse', 'GetServerCapabilitiesResponse', 'HealthResponse', 'ProtocolLimits', diff --git a/src/twinkle_client/types/base.py b/src/twinkle/protocol/types/base.py similarity index 98% rename from src/twinkle_client/types/base.py rename to src/twinkle/protocol/types/base.py index 19330327..d428afc8 100644 --- a/src/twinkle_client/types/base.py +++ b/src/twinkle/protocol/types/base.py @@ -2,7 +2,7 @@ """Shared pydantic base classes, field roles, and the naming rulings for the wire contract. This module is a public contract carrier imported across packages (Twinkle_Server -reverse-imports ``twinkle_client.types``); it therefore intentionally carries **no** +reverse-imports ``twinkle.protocol.types``); it therefore intentionally carries **no** underscore prefix. Naming rulings (authoritative for all three split specs; kept in code, not only in diff --git a/src/twinkle_client/types/checkpoint.py b/src/twinkle/protocol/types/checkpoint.py similarity index 100% rename from src/twinkle_client/types/checkpoint.py rename to src/twinkle/protocol/types/checkpoint.py diff --git a/src/twinkle_client/types/component.py b/src/twinkle/protocol/types/component.py similarity index 100% rename from src/twinkle_client/types/component.py rename to src/twinkle/protocol/types/component.py diff --git a/src/twinkle_client/types/data.py b/src/twinkle/protocol/types/data.py similarity index 98% rename from src/twinkle_client/types/data.py rename to src/twinkle/protocol/types/data.py index 9da24ac3..b7936dbc 100644 --- a/src/twinkle_client/types/data.py +++ b/src/twinkle/protocol/types/data.py @@ -14,7 +14,7 @@ reads is declared with a strict type (``StrictInt`` leaves reject ``true`` and ``1.0``), while unknown JSON-native keys are kept and re-exported: a user's preprocessor may leave extra columns on an entry and dropping them would lose - data the caller sent. See :class:`~twinkle_client.types.base.DataModel`. + data the caller sent. See :class:`~twinkle.protocol.types.base.DataModel`. - **Shallowest-first unions.** Nesting depth encodes tensor rank here, so a rank range needs a union. Declaring the deepest branch first is a large, silent pessimisation: given a 2-D input the 3-D branch does not fail at element 0, it @@ -43,7 +43,7 @@ from typing import Annotated, Any, List, Literal, Optional, Union from twinkle.data_format.encoding import ENCODED_INPUT_KEYS -from twinkle_client.types.base import DataModel +from twinkle.protocol.types.base import DataModel # --------------------------------------------------------------------------- # # Leaf types. Shallowest-first, and ``StrictInt`` wherever the values come from a diff --git a/src/twinkle_client/types/errors.py b/src/twinkle/protocol/types/errors.py similarity index 100% rename from src/twinkle_client/types/errors.py rename to src/twinkle/protocol/types/errors.py diff --git a/src/twinkle_client/types/lifecycle.py b/src/twinkle/protocol/types/lifecycle.py similarity index 97% rename from src/twinkle_client/types/lifecycle.py rename to src/twinkle/protocol/types/lifecycle.py index 911f7a0e..32555e24 100644 --- a/src/twinkle_client/types/lifecycle.py +++ b/src/twinkle/protocol/types/lifecycle.py @@ -2,7 +2,7 @@ """The request-lifecycle wire model: one envelope for submit and retrieve. This module is a public-contract carrier imported across packages (Twinkle_Server -reverse-imports ``twinkle_client.types``); per the naming rulings in ``base.py`` it +reverse-imports ``twinkle.protocol.types``); per the naming rulings in ``base.py`` it therefore intentionally carries **no** underscore prefix. """ from __future__ import annotations diff --git a/src/twinkle_client/types/model.py b/src/twinkle/protocol/types/model.py similarity index 90% rename from src/twinkle_client/types/model.py rename to src/twinkle/protocol/types/model.py index 40c1ffcd..b22d3164 100644 --- a/src/twinkle_client/types/model.py +++ b/src/twinkle/protocol/types/model.py @@ -3,7 +3,7 @@ One declaration per endpoint, shared by Twinkle_Client and the server handler, so there is a single answer to "what may this endpoint receive". Every field carries a -role (see :mod:`twinkle_client.types.base`): +role (see :mod:`twinkle.protocol.types.base`): - plain fields are **control** fields: the handler consumes them or passes them as a named argument, and they are never re-forwarded through ``**backend_kwargs``; @@ -180,9 +180,25 @@ class DataPlaneForwardRequest(StrictRequest): loss_kwargs: dict[str, JsonValue] = passthrough() -class DataPlaneForwardOnlyRequest(DataPlaneForwardRequest): - """Body of ``POST /twinkle/forward_only_from_data_plane``.""" +class DataPlaneForwardOnlyRequest(StrictRequest): + """Body of ``POST /twinkle/forward_only_from_data_plane``. + This endpoint is read-only, so it deliberately has no ``seq_id`` idempotency + key. Its fields are declared directly rather than inherited from the + gradient-mutating data-plane request. + """ + + input_refs: list[DataRef] = Field(min_length=1) + input_field: str | None = None + kwarg_fields: dict[str, str] = Field(default_factory=dict) + adapter_name: str + task: str | None = backend_kwarg(default=None) + temperature: float | None = backend_kwarg(default=None, gt=0) + return_logits: bool | None = backend_kwarg(default=None) + disable_lora: bool | None = backend_kwarg(default=None) + micro_batch_size: int | None = backend_kwarg(default=None, ge=1) + gradient_accumulation_steps: int | None = backend_kwarg(default=None, ge=1) + loss_kwargs: dict[str, JsonValue] = passthrough() output_ref: DataRef | None = None output_fields: dict[str, str] = Field(default_factory=dict) @@ -335,7 +351,7 @@ class OkResponse(ResponseModel): class ModelResult(ResponseModel): - """Generic single-value result wrapper returned by result-bearing endpoints.""" + """Generic result wrapper; ``ModelResult`` is the retained historical public name.""" result: Any @@ -385,58 +401,19 @@ class TrainingProgressResponse(ResponseModel): # --- Void responses (return None → OkResponse) --- - -class BackwardResponse(OkResponse): - """Response for /backward endpoint.""" - - -class StepResponse(OkResponse): - """Response for /step (optimizer step) endpoint.""" - - -class ZeroGradResponse(OkResponse): - """Response for /zero_grad endpoint.""" - - -class LrStepResponse(OkResponse): - """Response for /lr_step endpoint.""" - - -class SetLossResponse(OkResponse): - """Response for /set_loss endpoint.""" - - -class SetOptimizerResponse(OkResponse): - """Response for /set_optimizer endpoint.""" - - -class SetLrSchedulerResponse(OkResponse): - """Response for /set_lr_scheduler endpoint.""" - - -class LoadResponse(OkResponse): - """Response for /load endpoint.""" - - -class SetTemplateResponse(OkResponse): - """Response for /set_template endpoint.""" - - -class SetProcessorResponse(OkResponse): - """Response for /set_processor endpoint.""" - - -class ClipGradAndStepResponse(OkResponse): - """Response for /clip_grad_and_step endpoint.""" - - -class ApplyPatchResponse(OkResponse): - """Response for /apply_patch endpoint.""" - - -class AddMetricResponse(OkResponse): - """Response for /add_metric endpoint.""" - +BackwardResponse = OkResponse +StepResponse = OkResponse +ZeroGradResponse = OkResponse +LrStepResponse = OkResponse +SetLossResponse = OkResponse +SetOptimizerResponse = OkResponse +SetLrSchedulerResponse = OkResponse +LoadResponse = OkResponse +SetTemplateResponse = OkResponse +SetProcessorResponse = OkResponse +ClipGradAndStepResponse = OkResponse +ApplyPatchResponse = OkResponse +AddMetricResponse = OkResponse # --- Other responses --- diff --git a/src/twinkle_client/types/processor.py b/src/twinkle/protocol/types/processor.py similarity index 97% rename from src/twinkle_client/types/processor.py rename to src/twinkle/protocol/types/processor.py index f778f7ec..de7de0d9 100644 --- a/src/twinkle_client/types/processor.py +++ b/src/twinkle/protocol/types/processor.py @@ -14,7 +14,7 @@ that guesses. Class names are prefixed with ``Processor`` to avoid collisions when importing from -``twinkle_client.types`` alongside ``model.py``. +``twinkle.protocol.types`` alongside ``model.py``. """ from __future__ import annotations diff --git a/src/twinkle_client/types/sampler.py b/src/twinkle/protocol/types/sampler.py similarity index 95% rename from src/twinkle_client/types/sampler.py rename to src/twinkle/protocol/types/sampler.py index d69aa649..b89602b6 100644 --- a/src/twinkle_client/types/sampler.py +++ b/src/twinkle/protocol/types/sampler.py @@ -2,7 +2,7 @@ """Request / response models for the twinkle-native sampler endpoints. Shared by the server handler and the twinkle client. Field roles follow -:mod:`twinkle_client.types.base`; the sampler handlers pass everything they need +:mod:`twinkle.protocol.types.base`; the sampler handlers pass everything they need explicitly, so these requests carry control fields and -- for the template setter -- one passthrough region, and no free-floating backend kwargs. @@ -10,10 +10,10 @@ name (``AddAdapterRequest``, ``SetTemplateRequest``, ``CreateResponse`` and their responses). The two modules describe *different* endpoints with different field sets; a shared bare name is distinguished only by an import alias and, when a handler does -``import twinkle_client.types as types``, silently resolves to whichever module the +``import twinkle.protocol.types as types``, silently resolves to whichever module the package ``__init__`` re-exported first -- which is how the sampler endpoints once bound ``model.py``'s schema. Prefixing at the definition site removes the ambiguity, -matching :mod:`twinkle_client.types.processor`. +matching :mod:`twinkle.protocol.types.processor`. """ from __future__ import annotations diff --git a/src/twinkle_client/types/server.py b/src/twinkle/protocol/types/server.py similarity index 96% rename from src/twinkle_client/types/server.py rename to src/twinkle/protocol/types/server.py index e3d5159d..dd335d58 100644 --- a/src/twinkle_client/types/server.py +++ b/src/twinkle/protocol/types/server.py @@ -46,10 +46,6 @@ class DeleteCheckpointResponse(ResponseModel): message: str -class ErrorResponse(ResponseModel): - detail: str - - class WeightsInfoRequest(StrictRequest): twinkle_path: str diff --git a/src/twinkle_client/types/session.py b/src/twinkle/protocol/types/session.py similarity index 100% rename from src/twinkle_client/types/session.py rename to src/twinkle/protocol/types/session.py diff --git a/src/twinkle_client/types/training.py b/src/twinkle/protocol/types/training.py similarity index 100% rename from src/twinkle_client/types/training.py rename to src/twinkle/protocol/types/training.py diff --git a/src/twinkle/server/checkpoint/checkpoint_manager.py b/src/twinkle/server/checkpoint/checkpoint_manager.py index 538ea97b..773494b6 100644 --- a/src/twinkle/server/checkpoint/checkpoint_manager.py +++ b/src/twinkle/server/checkpoint/checkpoint_manager.py @@ -16,7 +16,7 @@ from twinkle import get_logger from twinkle.hub import HubOperation -from twinkle_client.types import ResolvedLoadPath +from twinkle.protocol.types import ResolvedLoadPath from .paths import CHECKPOINT_INFO_FILENAME, validate_user_path from .training_run_manager import BaseFileManager, BaseTrainingRunManager diff --git a/src/twinkle/server/checkpoint/twinkle.py b/src/twinkle/server/checkpoint/twinkle.py index 78eb2745..e4fedfc9 100644 --- a/src/twinkle/server/checkpoint/twinkle.py +++ b/src/twinkle/server/checkpoint/twinkle.py @@ -2,17 +2,17 @@ """ Twinkle-specific checkpoint and training-run managers. -Uses ``twinkle_client.types.training`` models for all serialization and response construction. +Uses ``twinkle.protocol.types.training`` models for all serialization and response construction. """ from datetime import datetime from typing import Any, Dict, List, Optional +from twinkle.protocol.types.training import (Checkpoint, CheckpointsListResponse, CreateModelRequest, Cursor, + ParsedCheckpointTwinklePath, TrainingRun, TrainingRunsResponse, + WeightsInfoResponse) from twinkle.server.checkpoint.checkpoint_manager import BaseCheckpointManager from twinkle.server.checkpoint.paths import TRAIN_RUN_INFO_FILENAME, validate_ownership from twinkle.server.checkpoint.training_run_manager import BaseTrainingRunManager -from twinkle_client.types.training import (Checkpoint, CheckpointsListResponse, CreateModelRequest, Cursor, - ParsedCheckpointTwinklePath, TrainingRun, TrainingRunsResponse, - WeightsInfoResponse) class TwinkleTrainingRunManager(BaseTrainingRunManager): diff --git a/src/twinkle/server/config/server_config.py b/src/twinkle/server/config/server_config.py index fd5da6c4..82bac4d4 100644 --- a/src/twinkle/server/config/server_config.py +++ b/src/twinkle/server/config/server_config.py @@ -71,8 +71,6 @@ def from_yaml(cls, path: str | Path) -> ServerConfig: def _validate_cross_field(self) -> ServerConfig: if self.persistence.mode == 'redis' and not self.persistence.redis_url: raise ValueError("persistence.redis_url is required when persistence.mode == 'redis'", ) - if self.persistence.mode == 'file' and not self.persistence.file_path: - raise ValueError("persistence.file_path is required when persistence.mode == 'file'", ) return self # ---- round-trip / serialization -------------------------------------- # diff --git a/src/twinkle/server/data_plane/handlers.py b/src/twinkle/server/data_plane/handlers.py index aaf214ed..508b4dde 100644 --- a/src/twinkle/server/data_plane/handlers.py +++ b/src/twinkle/server/data_plane/handlers.py @@ -5,7 +5,7 @@ from fastapi import Depends, FastAPI from typing import TYPE_CHECKING -import twinkle_client.types as types +import twinkle.protocol.types as types if TYPE_CHECKING: from .app import DataPlaneManagement diff --git a/src/twinkle/server/data_plane/proxy.py b/src/twinkle/server/data_plane/proxy.py index 0aa2ae7f..3d1aa97d 100644 --- a/src/twinkle/server/data_plane/proxy.py +++ b/src/twinkle/server/data_plane/proxy.py @@ -5,8 +5,8 @@ import httpx from typing import Any -from twinkle_client.http.headers import build_routing_headers -from twinkle_client.types.component import DataRef +from twinkle.protocol.headers import build_routing_headers +from twinkle.protocol.types.component import DataRef class DataPlaneProxy: diff --git a/src/twinkle/server/data_plane/store.py b/src/twinkle/server/data_plane/store.py index ad8a66fe..291a6deb 100644 --- a/src/twinkle/server/data_plane/store.py +++ b/src/twinkle/server/data_plane/store.py @@ -6,8 +6,8 @@ from typing import Any from twinkle.data_format import rows_to_tq_fields -from twinkle_client.common.json_utils import json_safe -from twinkle_client.types.component import DataRef +from twinkle.protocol.json_utils import json_safe +from twinkle.protocol.types.component import DataRef def _keys(ref: DataRef) -> list[str]: diff --git a/src/twinkle/server/deployment.py b/src/twinkle/server/deployment.py index 792ae6d3..cdf2f868 100644 --- a/src/twinkle/server/deployment.py +++ b/src/twinkle/server/deployment.py @@ -36,12 +36,12 @@ from ray import serve from typing import Any +from twinkle.protocol.types.errors import ErrorCategory, ErrorPayload from twinkle.server.exceptions import TwinkleServerError from twinkle.server.middleware.auth import verify_request_token from twinkle.server.telemetry.http_middleware import create_metrics_middleware from twinkle.server.telemetry.tracing import create_tracing_middleware from twinkle.utils.logger import get_logger -from twinkle_client.types.errors import ErrorCategory, ErrorPayload logger = get_logger() diff --git a/src/twinkle/server/exceptions.py b/src/twinkle/server/exceptions.py index 0d8c6437..b8985879 100644 --- a/src/twinkle/server/exceptions.py +++ b/src/twinkle/server/exceptions.py @@ -9,7 +9,7 @@ from __future__ import annotations -from twinkle_client.types.errors import ErrorCategory +from twinkle.protocol.types.errors import ErrorCategory class TwinkleServerError(Exception): diff --git a/src/twinkle/server/gateway/app.py b/src/twinkle/server/gateway/app.py index 74070e40..0c54f506 100644 --- a/src/twinkle/server/gateway/app.py +++ b/src/twinkle/server/gateway/app.py @@ -11,7 +11,7 @@ from fastapi import FastAPI, HTTPException from typing import Any -import twinkle_client.types as types +import twinkle.protocol.types as types from twinkle.server.deployment import LazyCleanupMixin, bind_deployment, build_deployment_app from twinkle.server.state import get_server_state from twinkle.utils.logger import get_logger diff --git a/src/twinkle/server/gateway/openai_handlers.py b/src/twinkle/server/gateway/openai_handlers.py index 11030e06..aeed5195 100644 --- a/src/twinkle/server/gateway/openai_handlers.py +++ b/src/twinkle/server/gateway/openai_handlers.py @@ -15,7 +15,7 @@ from fastapi.responses import JSONResponse, StreamingResponse from typing import TYPE_CHECKING, Any -from twinkle_client.http.headers import H_AUTH, H_AUTH_TWINKLE, build_routing_headers +from twinkle.protocol.headers import H_AUTH, H_AUTH_TWINKLE, build_routing_headers if TYPE_CHECKING: from .app import GatewayServer diff --git a/src/twinkle/server/gateway/proxy.py b/src/twinkle/server/gateway/proxy.py index c12c5bc4..664d68d2 100644 --- a/src/twinkle/server/gateway/proxy.py +++ b/src/twinkle/server/gateway/proxy.py @@ -13,10 +13,10 @@ from fastapi.responses import JSONResponse from typing import Any +from twinkle.protocol.headers import H_MULTIPLEX, H_MULTIPLEX_LEGACY, H_REQUEST_ID, H_REQUEST_ID_LEGACY +from twinkle.protocol.types.errors import ErrorCategory, ErrorPayload from twinkle.server.telemetry.tracing import inject_context from twinkle.utils.logger import get_logger -from twinkle_client.http.headers import H_MULTIPLEX, H_MULTIPLEX_LEGACY, H_REQUEST_ID, H_REQUEST_ID_LEGACY -from twinkle_client.types.errors import ErrorCategory, ErrorPayload from . import routes logger = get_logger() diff --git a/src/twinkle/server/gateway/tinker_handlers.py b/src/twinkle/server/gateway/tinker_handlers.py index a2dfd0dd..93809d78 100644 --- a/src/twinkle/server/gateway/tinker_handlers.py +++ b/src/twinkle/server/gateway/tinker_handlers.py @@ -82,7 +82,7 @@ async def get_server_capabilities( request: Request, self: GatewayServer = Depends(self_fn), ) -> types.GetServerCapabilitiesResponse: - # Convert twinkle_client.types.SupportedModel to tinker.types.SupportedModel + # Convert twinkle.protocol.types.SupportedModel to tinker.types.SupportedModel tinker_supported_models = [types.SupportedModel(model_name=m.model_name) for m in self.supported_models] return types.GetServerCapabilitiesResponse(supported_models=tinker_supported_models) diff --git a/src/twinkle/server/gateway/twinkle_handlers.py b/src/twinkle/server/gateway/twinkle_handlers.py index a78f7730..3df44ffc 100644 --- a/src/twinkle/server/gateway/twinkle_handlers.py +++ b/src/twinkle/server/gateway/twinkle_handlers.py @@ -13,7 +13,7 @@ if TYPE_CHECKING: from .app import GatewayServer -import twinkle_client.types as types +import twinkle.protocol.types as types from twinkle.server.checkpoint import create_checkpoint_manager, create_training_run_manager, validate_user_path from twinkle.server.exceptions import RequestRejectedError, ResourceNotFoundError from twinkle.server.lifecycle.envelope import envelope_from_record diff --git a/src/twinkle/server/lifecycle/envelope.py b/src/twinkle/server/lifecycle/envelope.py index 85274084..25343c9b 100644 --- a/src/twinkle/server/lifecycle/envelope.py +++ b/src/twinkle/server/lifecycle/envelope.py @@ -10,10 +10,10 @@ from typing import Any +from twinkle.protocol.types.errors import ErrorCategory, ErrorPayload +from twinkle.protocol.types.lifecycle import TaskEnvelope from twinkle.server.state.models import FutureFailureRecord from twinkle.server.task_errors import trim_traceback -from twinkle_client.types.errors import ErrorCategory, ErrorPayload -from twinkle_client.types.lifecycle import TaskEnvelope # Keys must equal ``state.models.FAILURE_REASON_CODES`` (guarded by test_envelope). _FAILURE_WIRE: dict[str, tuple[int, ErrorCategory]] = { diff --git a/src/twinkle/server/lifecycle/submit.py b/src/twinkle/server/lifecycle/submit.py index cc767fd5..b1b409a4 100644 --- a/src/twinkle/server/lifecycle/submit.py +++ b/src/twinkle/server/lifecycle/submit.py @@ -13,11 +13,11 @@ from typing import TYPE_CHECKING, Any from twinkle.data_format import InputFeature, Trajectory, is_encoded +from twinkle.protocol.types.base import FieldRole, fields_with_role +from twinkle.protocol.types.data import export_batch +from twinkle.protocol.types.lifecycle import TaskEnvelope from twinkle.server.middleware.auth import get_session_id_from_request from twinkle.server.validation import assert_request_supported -from twinkle_client.types.base import FieldRole, fields_with_role -from twinkle_client.types.data import export_batch -from twinkle_client.types.lifecycle import TaskEnvelope if TYPE_CHECKING: from twinkle.server.lifecycle.protocols import DataParallelDeployment, QueuedDeployment @@ -31,7 +31,7 @@ def to_backend_inputs(inputs: Any, *, single: bool = False) -> Any: """Seam A: export wire-validated ``inputs`` as the objects the backend consumes. This is an *export*, not a validation step. The request model declares ``inputs`` - as :data:`~twinkle_client.types.data.WireInputBatch`, so a malformed batch is + as :data:`~twinkle.protocol.types.data.WireInputBatch`, so a malformed batch is already rejected during FastAPI body parsing -- before a future record exists and before anything reaches a GPU. Validating here instead would put the first check inside the queued task, where a rejection has already cost an enqueue. @@ -71,7 +71,7 @@ def backend_kwargs(body: Any) -> dict[str, Any]: """Seam B: the keyword arguments forwarded to the backend call. Exactly two sources, both declared on the request model (see - :mod:`twinkle_client.types.base`): + :mod:`twinkle.protocol.types.base`): 1. fields whose role is ``BackendKwarg``, included iff their value is not ``None``; 2. the contents of every ``Passthrough`` field, flattened. diff --git a/src/twinkle/server/middleware/auth.py b/src/twinkle/server/middleware/auth.py index ef779cd2..c3b7f245 100644 --- a/src/twinkle/server/middleware/auth.py +++ b/src/twinkle/server/middleware/auth.py @@ -3,7 +3,7 @@ from fastapi.responses import JSONResponse from typing import Any -from twinkle_client.http.headers import H_AUTH, H_AUTH_TWINKLE, H_REQUEST_ID +from twinkle.protocol.headers import H_AUTH, H_AUTH_TWINKLE, H_REQUEST_ID _OPENAI_COMPAT_SUFFIXES = ('/chat/completions', '/models') diff --git a/src/twinkle/server/model/data_plane_inputs.py b/src/twinkle/server/model/data_plane_inputs.py index aed4b3c2..fd0c6e22 100644 --- a/src/twinkle/server/model/data_plane_inputs.py +++ b/src/twinkle/server/model/data_plane_inputs.py @@ -5,7 +5,7 @@ import asyncio from typing import Any -from twinkle_client.common.json_utils import json_safe +from twinkle.protocol.json_utils import json_safe def model_result_rows(result: Any, batch_size: int) -> list[dict[str, Any]]: diff --git a/src/twinkle/server/model/twinkle_handlers.py b/src/twinkle/server/model/twinkle_handlers.py index adc548f2..a63a5bfb 100644 --- a/src/twinkle/server/model/twinkle_handlers.py +++ b/src/twinkle/server/model/twinkle_handlers.py @@ -20,7 +20,8 @@ if TYPE_CHECKING: from .app import ModelManagement -import twinkle_client.types as types +import twinkle.protocol.types as types +from twinkle.protocol.serialize import deserialize_object from twinkle.server.checkpoint import (_resolve_client_save_dir, create_checkpoint_manager, create_training_run_manager, validate_user_path) from twinkle.server.exceptions import RequestRejectedError, TrainModeMismatchError @@ -409,7 +410,6 @@ async def add_metric(request: Request, body: types.AddMetricRequest, self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: async def _call(self, body, adapter_name, token): - from twinkle_client.common.serialize import deserialize_object metric_cls = deserialize_object(body.metric_cls) await self.call_backend( self.model.add_metric, @@ -426,7 +426,6 @@ async def apply_patch( self: ModelManagement = Depends(self_fn)) -> types.TaskEnvelope: async def _call(self, body, adapter_name, token): - from twinkle_client.common.serialize import deserialize_object patch_cls = deserialize_object(body.patch_cls) await self.call_backend( self.model.apply_patch, @@ -560,7 +559,6 @@ async def add_adapter_to_model( except ValueError as exc: raise RequestRejectedError(str(exc)) from exc - from twinkle_client.common.serialize import deserialize_object config = deserialize_object(body.config) # ---- Decision_Boundary left: validate against the deployment's train_mode ---- diff --git a/src/twinkle/server/processor/twinkle_handlers.py b/src/twinkle/server/processor/twinkle_handlers.py index 70645f9f..5de7814e 100644 --- a/src/twinkle/server/processor/twinkle_handlers.py +++ b/src/twinkle/server/processor/twinkle_handlers.py @@ -19,7 +19,8 @@ if TYPE_CHECKING: from .app import ProcessorManagement -import twinkle_client.types as types +import twinkle.protocol.types as types +from twinkle.protocol.serialize import deserialize_object from twinkle.server.middleware.auth import get_session_id_from_request, get_token_from_request from twinkle.server.telemetry.correlation import SESSION_ID, TOKEN_ID from twinkle.server.telemetry.tracing import traced_operation @@ -71,7 +72,6 @@ async def create( _kwargs.pop('remote_group', None) _kwargs.pop('device_mesh', None) - from twinkle_client.common.serialize import deserialize_object resolved_kwargs = {} for key, value in _kwargs.items(): if isinstance(value, str) and value.startswith('pid:'): @@ -132,7 +132,6 @@ async def call( assert function is not None, f'`{function_name}` not found in {processor.__class__}' assert hasattr(function, '_execute'), f'Cannot call inner method of {processor.__class__}' - from twinkle_client.common.serialize import deserialize_object resolved_kwargs = {} for key, value in _kwargs.items(): if isinstance(value, str) and value.startswith('pid:'): diff --git a/src/twinkle/server/sampler/twinkle_handlers.py b/src/twinkle/server/sampler/twinkle_handlers.py index de8c9ffb..2aca4b22 100644 --- a/src/twinkle/server/sampler/twinkle_handlers.py +++ b/src/twinkle/server/sampler/twinkle_handlers.py @@ -22,8 +22,11 @@ import numpy as np -import twinkle_client.types as types +import twinkle.protocol.types as types from twinkle.data_format import SamplingParams +from twinkle.protocol.json_utils import json_safe +from twinkle.protocol.serialize import deserialize_object +from twinkle.protocol.types import sampler as sampler_types from twinkle.server.exceptions import EndpointUnavailableError, RequestRejectedError from twinkle.server.lifecycle.submit import backend_kwargs, resolve_twinkle_adapter_name, to_backend_inputs from twinkle.server.sampler.weights import resolve_sampler_weights @@ -31,8 +34,6 @@ from twinkle.server.telemetry.correlation import MODEL_ID from twinkle.server.telemetry.tracing import traced_operation from twinkle.utils.logger import get_logger -from twinkle_client.common.json_utils import json_safe -from twinkle_client.types import sampler as sampler_types logger = get_logger() @@ -377,7 +378,6 @@ async def apply_patch( body: types.ApplyPatchRequest, self: SamplerManagement = Depends(self_fn), ) -> None: - from twinkle_client.common.serialize import deserialize_object patch_cls = deserialize_object(body.patch_cls) with traced_operation('sampler.apply_patch'): await self.call_backend(self.sampler.apply_patch, patch_cls, **backend_kwargs(body)) diff --git a/src/twinkle/server/session_resource/base.py b/src/twinkle/server/session_resource/base.py index 4411f12b..4294da51 100644 --- a/src/twinkle/server/session_resource/base.py +++ b/src/twinkle/server/session_resource/base.py @@ -205,16 +205,6 @@ def get_resource_state(self, resource_id: str, key: str, default: Any = None) -> state = info.get('state') or {} return state.get(key, default) - def pop_resource_state(self, resource_id: str, key: str, default: Any = None) -> Any: - """Pop a per-resource state value.""" - info = self._resource_records.get(resource_id) - if info is None: - return default - state = info.get('state') - if not isinstance(state, dict): - return default - return state.pop(key, default) - def clear_resource_state(self, resource_id: str) -> None: """Clear all per-resource state values.""" info = self._resource_records.get(resource_id) @@ -350,10 +340,6 @@ def _ensure_countdown_started(self) -> None: self._countdown_task = asyncio.create_task(self._resource_countdown_loop()) logger.debug(f'[{self._resource_type}Manager] Countdown task started') - async def _async_ensure_countdown_started(self) -> None: - """Async version for convenience.""" - self._ensure_countdown_started() - def stop_resource_countdown(self) -> None: """Stop the background countdown task.""" if self._resource_countdown_running: diff --git a/src/twinkle/server/state/__init__.py b/src/twinkle/server/state/__init__.py index 9acc8aca..5f9a4754 100644 --- a/src/twinkle/server/state/__init__.py +++ b/src/twinkle/server/state/__init__.py @@ -2,7 +2,6 @@ from .backend import create_backend from .base import BaseManager from .cleanup_coordinator import ResourceCleanupCoordinator -from .config_manager import ConfigManager from .count_publisher import ResourceCountPublisher from .future_manager import FutureManager from .model_manager import ModelManager @@ -25,7 +24,6 @@ 'ModelManager', 'SamplingSessionManager', 'FutureManager', - 'ConfigManager', # Server state 'ServerState', 'ReplicaRegistry', diff --git a/src/twinkle/server/state/backend/base.py b/src/twinkle/server/state/backend/base.py index f801335c..b65f7d2b 100644 --- a/src/twinkle/server/state/backend/base.py +++ b/src/twinkle/server/state/backend/base.py @@ -4,8 +4,8 @@ from collections.abc import Callable from typing import Any +from twinkle.protocol.types.errors import ErrorCategory from twinkle.server.exceptions import StateBackendError -from twinkle_client.types.errors import ErrorCategory class ConcurrencyError(StateBackendError): @@ -93,13 +93,10 @@ async def update_atomic( """ ... + @abstractmethod async def mget(self, keys: list[str]) -> list[Any | None]: - """Batch-read multiple keys. Returns values in the same order as *keys*. - - Default implementation falls back to serial ``get()`` calls. - Backends should override for efficiency (e.g. Redis MGET). - """ - return [await self.get(key) for key in keys] + """Batch-read multiple keys, preserving input order.""" + ... @abstractmethod async def close(self) -> None: diff --git a/src/twinkle/server/state/config_manager.py b/src/twinkle/server/state/config_manager.py deleted file mode 100644 index b5a3185a..00000000 --- a/src/twinkle/server/state/config_manager.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) ModelScope Contributors. All rights reserved. -from __future__ import annotations - -from typing import Any - -from .backend import StateBackend - -# Key prefix used to namespace configuration entries inside the backend. -_CONFIG_PREFIX = 'config::' -_CONFIG_PATTERN = f'{_CONFIG_PREFIX}*' - - -class ConfigManager: - """ - Manages key-value configuration entries via a :class:`StateBackend`. - - Configuration entries have no expiry; they persist until explicitly removed - or cleared. This manager does not inherit from BaseManager because config - values are arbitrary Python objects rather than Pydantic models, and all - storage is delegated to the injected backend. - - Methods are ``async`` because :class:`StateBackend` operations are async. - Atomicity for read-modify-write entries comes from the backend's own - primitives (``set_nx`` / ``update_atomic``), not from any single-threaded - actor assumption — each worker holds its own ``ConfigManager`` bound to the - shared backend, so no additional locking is layered on top of the backend. - """ - - def __init__(self, backend: StateBackend) -> None: - self._backend = backend - - @staticmethod - def _make_key(key: str) -> str: - return f'{_CONFIG_PREFIX}{key}' - - # ----- CRUD ----- - - async def add(self, key: str, value: Any) -> None: - """Add or overwrite a configuration value.""" - await self._backend.set(self._make_key(key), value) - - async def add_or_get(self, key: str, value: Any) -> Any: - """Add a value if the key does not exist; otherwise return the existing value. - - Args: - key: Configuration key. - value: Value to store if the key is absent. - - Returns: - The existing or newly stored value. - """ - backend_key = self._make_key(key) - existing = await self._backend.get(backend_key) - if existing is not None: - return existing - # Use set_nx for atomicity within a single backend; if another - # writer already populated the key we return the winning value. - if await self._backend.set_nx(backend_key, value): - return value - return await self._backend.get(backend_key) - - async def get(self, key: str) -> Any | None: - """Return the configuration value for key, or None.""" - return await self._backend.get(self._make_key(key)) - - async def pop(self, key: str) -> Any | None: - """Remove and return the configuration value for key, or None. - - Note: get-then-delete is not atomic (TOCTOU); a concurrent pop may - return the same value twice. This is acceptable for config entries - where double-return is harmless. - """ - backend_key = self._make_key(key) - value = await self._backend.get(backend_key) - if value is None: - return None - await self._backend.delete(backend_key) - return value - - async def clear(self) -> None: - """Remove all configuration entries.""" - keys = await self._backend.keys(_CONFIG_PATTERN) - for backend_key in keys: - await self._backend.delete(backend_key) - - async def count(self) -> int: - """Return the number of stored configuration entries.""" - return await self._backend.count(_CONFIG_PATTERN) diff --git a/src/twinkle/server/state/model_manager.py b/src/twinkle/server/state/model_manager.py index 26e7b4c0..aaf520fc 100644 --- a/src/twinkle/server/state/model_manager.py +++ b/src/twinkle/server/state/model_manager.py @@ -232,12 +232,6 @@ async def cleanup_expired(self, cutoff_time: float, expired_session_ids: list[st # ----- Backend-derived helpers --------------------------------------- # - async def _count_models_for_token(self, token: str | None) -> int: - if not token: - return 0 - all_records = await self.get_all() - return sum(1 for r in all_records.values() if r.token == token) - async def _models_for_replica(self, replica_id: str) -> list[str]: all_records = await self.get_all() return [mid for mid, r in all_records.items() if r.replica_id == replica_id] diff --git a/src/twinkle/server/state/replica_registry.py b/src/twinkle/server/state/replica_registry.py index d3eda8d8..ae1118dd 100644 --- a/src/twinkle/server/state/replica_registry.py +++ b/src/twinkle/server/state/replica_registry.py @@ -48,16 +48,6 @@ async def touch_last_seen(self, replica_id: str) -> None: """Refresh the replica's liveness timestamp (separate key from max_loras).""" await self._backend.set(_last_seen_key(replica_id), time.time()) - async def get_last_seen(self, replica_id: str) -> float | None: - """Return the replica's last-seen unix time, or ``None`` if never refreshed.""" - value = await self._backend.get(_last_seen_key(replica_id)) - if value is None: - return None - try: - return float(value) - except (TypeError, ValueError): - return None - async def get_all_last_seen(self) -> dict[str, float]: """Return every replica's last-seen timestamp.""" keys = await self._backend.keys(f'{REPLICA_PREFIX}*{_LAST_SEEN_SUFFIX}') @@ -73,16 +63,6 @@ async def get_all_last_seen(self) -> dict[str, float]: continue return out - async def get_max_loras(self, replica_id: str) -> int | None: - """Return the declared capacity, or ``None`` if the replica is unknown.""" - value = await self._backend.get(_make_key(replica_id)) - if value is None: - return None - try: - return int(value) - except (TypeError, ValueError): - return None - async def get_all(self) -> dict[str, int]: """Return every registered replica's declared capacity.""" keys = await self._backend.keys(f'{REPLICA_PREFIX}*{_MAX_LORAS_SUFFIX}') diff --git a/src/twinkle/server/state/server_state.py b/src/twinkle/server/state/server_state.py index daadc051..dd926aa5 100644 --- a/src/twinkle/server/state/server_state.py +++ b/src/twinkle/server/state/server_state.py @@ -18,7 +18,6 @@ from .backend import StateBackend from .backend.factory import create_backend from .cleanup_coordinator import ResourceCleanupCoordinator -from .config_manager import ConfigManager from .count_publisher import ResourceCountPublisher from .future_manager import FutureManager from .model_manager import ModelManager @@ -95,13 +94,12 @@ def _sweep_processor_transform(existing: Any, *, now: float) -> dict[str, dict[s class ServerState: """Unified server state management class. - Composes five resource managers: + Composes four resource managers: - :class:`SessionManager` — client sessions - :class:`ModelManager` — registered models - :class:`SamplingSessionManager` — sampling sessions - :class:`FutureManager` — async task futures - - :class:`ConfigManager` — key-value configuration Each Ray Serve worker owns one process-local instance, bound directly to a shared :class:`StateBackend`. @@ -133,7 +131,6 @@ def __init__( self._model_mgr = ModelManager(self._backend, expiration_timeout, per_token_model_limit) self._sampling_mgr = SamplingSessionManager(self._backend, expiration_timeout) self._future_mgr = FutureManager(self._backend, expiration_timeout) - self._config_mgr = ConfigManager(self._backend) self.expiration_timeout = expiration_timeout self.cleanup_interval = cleanup_interval @@ -506,32 +503,6 @@ async def store_future_status( absolute_deadline=absolute_deadline, ) - # ----- Configuration Management ----- - - async def add_config(self, key: str, value: Any) -> None: - """Add or overwrite a configuration value.""" - await self._config_mgr.add(key, value) - - async def add_or_get_config(self, key: str, value: Any) -> Any: - """Add a config value if absent; otherwise return the existing value.""" - return await self._config_mgr.add_or_get(key, value) - - async def get_config(self, key: str) -> Any | None: - """Return the configuration value for key, or None.""" - return await self._config_mgr.get(key) - - async def pop_config(self, key: str) -> Any | None: - """Remove and return the configuration value for key, or None.""" - return await self._config_mgr.pop(key) - - async def clear_config(self) -> None: - """Remove all configuration entries.""" - await self._config_mgr.clear() - - async def count_config(self) -> int: - """Return the number of stored configuration entries.""" - return await self._config_mgr.count() - # ----- Resource Cleanup ----- async def cleanup_expired_resources(self) -> dict[str, int]: diff --git a/src/twinkle/server/task_errors.py b/src/twinkle/server/task_errors.py index 2ad62111..13402611 100644 --- a/src/twinkle/server/task_errors.py +++ b/src/twinkle/server/task_errors.py @@ -4,7 +4,7 @@ from typing import Any -from twinkle_client.types.errors import ErrorCategory, ErrorPayload +from twinkle.protocol.types.errors import ErrorCategory, ErrorPayload _ERROR_MAX = 1024 _TRACEBACK_MAX = 65536 diff --git a/src/twinkle/server/task_queue/config.py b/src/twinkle/server/task_queue/config.py index 71a9a8a0..449c694c 100644 --- a/src/twinkle/server/task_queue/config.py +++ b/src/twinkle/server/task_queue/config.py @@ -21,8 +21,8 @@ class TaskQueueConfig(BaseModel): """Configuration for task queue and rate limiting. Attributes: - rps_limit: Maximum requests per second per user token. ``0`` disables. - tps_limit: Maximum input tokens per second per user token. ``0`` disables. + rps_limit: Maximum requests per second per user token, per replica. ``0`` disables. + tps_limit: Maximum input tokens per second per user token, per replica. ``0`` disables. window_seconds: Sliding window for rate-limit calculations. Must be > 0. queue_timeout: Maximum time a task can wait in queue (seconds). execution_timeout: Maximum time a task can execute (seconds). ``0`` means "no diff --git a/src/twinkle/server/task_queue/mixin.py b/src/twinkle/server/task_queue/mixin.py index 0d89d4ff..31bdff26 100644 --- a/src/twinkle/server/task_queue/mixin.py +++ b/src/twinkle/server/task_queue/mixin.py @@ -14,13 +14,13 @@ from collections.abc import Callable, Coroutine from typing import TYPE_CHECKING, Any +from twinkle.protocol.types.lifecycle import TERMINAL_STATUSES, TaskEnvelope from twinkle.server.exceptions import BatchSizeError, ConfigError, InputTokensExceededError, RateLimitExceededError from twinkle.server.lifecycle.envelope import envelope_from_record from twinkle.server.lifecycle.poll_config import long_poll_window from twinkle.server.state.models import FutureFailureRecord from twinkle.server.telemetry.metrics import get_task_metrics from twinkle.utils.logger import get_logger -from twinkle_client.types.lifecycle import TERMINAL_STATUSES, TaskEnvelope from .backend_gate import BackendGate from .config import TaskQueueConfig from .rate_limiter import RateLimiter @@ -54,6 +54,7 @@ class TaskQueueMixin: """ state: ServerState + replica_id: str def _init_task_queue( self, @@ -108,6 +109,7 @@ def _init_task_queue( token_cleanup_interval=self._task_queue_config.token_cleanup_interval, active_tokens_gauge=self._task_metrics.rate_limiter_active_tokens if self._task_metrics else None, deployment_name=deployment_name, + replica_id=self.replica_id, ) self._rate_limiter.start_cleanup_task() diff --git a/src/twinkle/server/task_queue/rate_limiter.py b/src/twinkle/server/task_queue/rate_limiter.py index 7e301674..eebb72a3 100644 --- a/src/twinkle/server/task_queue/rate_limiter.py +++ b/src/twinkle/server/task_queue/rate_limiter.py @@ -10,7 +10,6 @@ import asyncio import time -from typing import Any from twinkle.utils.logger import get_logger @@ -43,12 +42,15 @@ def __init__( token_cleanup_interval: float = 60.0, active_tokens_gauge=None, deployment_name: str = '', + replica_id: str = '', ): """Initialize the rate limiter. Args: - rps_limit: Maximum requests per second per user token. - tps_limit: Maximum input tokens per second per user token. + rps_limit: Maximum requests per second per user token, per replica. + ``0`` disables the RPS limit. + tps_limit: Maximum input tokens per second per user token, per replica. + ``0`` disables the TPS limit. window_seconds: Time window for rate limiting (default 1.0s). token_cleanup_multiplier: Multiplier for token cleanup threshold. Tokens inactive for window_seconds * token_cleanup_multiplier @@ -58,6 +60,7 @@ def __init__( active_tokens_gauge: Optional gauge adapter (see twinkle.server.telemetry.metrics) for tracking the active token count. deployment_name: Deployment name for metrics labels. + replica_id: Replica identifier for metrics labels. """ self.rps_limit = rps_limit self.tps_limit = tps_limit @@ -79,7 +82,11 @@ def __init__( # Metrics gauge for active token count self._active_tokens_gauge = active_tokens_gauge - self._deployment_name = deployment_name + self._metric_tags = {} + if deployment_name: + self._metric_tags['deployment'] = deployment_name + if replica_id: + self._metric_tags['replica'] = replica_id def _cleanup_old_requests(self, token: str, current_time: float) -> None: """Remove requests outside the sliding window.""" @@ -120,8 +127,7 @@ async def _cleanup_inactive_tokens(self) -> None: f'Active tokens remaining: {len(self._token_requests)}') if self._active_tokens_gauge is not None: - tags = {'deployment': self._deployment_name} if self._deployment_name else {} - self._active_tokens_gauge.set(len(self._token_requests), tags=tags) + self._active_tokens_gauge.set(len(self._token_requests), tags=self._metric_tags) except asyncio.CancelledError: logger.debug('[RateLimiter] Cleanup task cancelled') @@ -165,45 +171,13 @@ async def check_and_record(self, token: str, input_tokens: int) -> tuple[bool, s request_count = len(requests) token_count = sum(count for _, count in requests) - if request_count >= self.rps_limit: + if self.rps_limit > 0 and request_count >= self.rps_limit: return False, f'RPS limit exceeded: {request_count}/{self.rps_limit} requests/s' - if token_count + input_tokens > self.tps_limit: + if self.tps_limit > 0 and token_count + input_tokens > self.tps_limit: return False, f'TPS limit exceeded: {token_count + input_tokens}/{self.tps_limit} tokens/s' self._token_requests[token].append((current_time, input_tokens)) if self._active_tokens_gauge is not None: - tags = {'deployment': self._deployment_name} if self._deployment_name else {} - self._active_tokens_gauge.set(len(self._token_requests), tags=tags) + self._active_tokens_gauge.set(len(self._token_requests), tags=self._metric_tags) return True, None - - def get_stats(self, token: str) -> dict[str, Any]: - """Get current rate limiting stats for a token.""" - current_time = time.time() - self._cleanup_old_requests(token, current_time) - - if token in self._token_requests: - self._last_activity[token] = current_time - - requests = self._token_requests.get(token, []) - request_count = len(requests) - token_count = sum(count for _, count in requests) - - return { - 'current_rps': request_count, - 'current_tps': token_count, - 'rps_limit': self.rps_limit, - 'tps_limit': self.tps_limit, - 'rps_available': self.rps_limit - request_count, - 'tps_available': self.tps_limit - token_count, - } - - def get_memory_stats(self) -> dict[str, Any]: - """Get memory usage statistics for monitoring.""" - return { - 'active_tokens': len(self._token_requests), - 'tracked_tokens': len(self._last_activity), - 'cleanup_threshold_seconds': self.window_seconds * self.token_cleanup_multiplier, - 'cleanup_interval_seconds': self.token_cleanup_interval, - 'cleanup_task_running': self._cleanup_started and self._cleanup_task and not self._cleanup_task.done(), - } diff --git a/src/twinkle/server/task_queue/types.py b/src/twinkle/server/task_queue/types.py index b173b0c1..9085ff8e 100644 --- a/src/twinkle/server/task_queue/types.py +++ b/src/twinkle/server/task_queue/types.py @@ -60,4 +60,3 @@ class QueuedTask: input_tokens: int task_type: str | None created_at: float - first_rate_limited_at: float | None = None diff --git a/src/twinkle/server/task_queue/worker.py b/src/twinkle/server/task_queue/worker.py index aba5145c..aaa9a6b9 100644 --- a/src/twinkle/server/task_queue/worker.py +++ b/src/twinkle/server/task_queue/worker.py @@ -14,6 +14,7 @@ from collections import deque from typing import TYPE_CHECKING, Any, Callable, Deque +from twinkle.protocol.types.errors import ErrorCategory from twinkle.server.exceptions import (BatchSizeError, EndpointUnavailableError, FullModeBusyError, InputTokensExceededError, RateLimitExceededError, RequestRejectedError, ResourceNotFoundError, ResourceQuotaExceededError, StateBackendError, @@ -23,7 +24,6 @@ from twinkle.server.telemetry.correlation import MODEL_ID, TOKEN_ID from twinkle.server.telemetry.tracing import traced_operation from twinkle.utils.logger import get_logger -from twinkle_client.types.errors import ErrorCategory from .config import TaskQueueConfig from .types import BackendBusyError, QueuedTask, QueueState, TaskStatus, UserTaskError diff --git a/src/twinkle/server/validation/backend_compat.py b/src/twinkle/server/validation/backend_compat.py index 074bb0a4..3c3d4b42 100644 --- a/src/twinkle/server/validation/backend_compat.py +++ b/src/twinkle/server/validation/backend_compat.py @@ -28,9 +28,9 @@ from enum import StrEnum from typing import Any, Optional +from twinkle.protocol.types.base import FieldRole, fields_with_role, read_backend_only from twinkle.server.exceptions import EndpointUnavailableError, RequestRejectedError from twinkle.utils.logger import get_logger -from twinkle_client.types.base import FieldRole, fields_with_role, read_backend_only class BackendCapability(StrEnum): diff --git a/src/twinkle/utils/import_utils.py b/src/twinkle/utils/import_utils.py index d460521d..e394aa5e 100644 --- a/src/twinkle/utils/import_utils.py +++ b/src/twinkle/utils/import_utils.py @@ -1,13 +1,7 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -import importlib import importlib.metadata -import importlib.util -import os from functools import lru_cache -from itertools import chain from packaging.requirements import Requirement -from types import ModuleType -from typing import Any @lru_cache @@ -31,57 +25,3 @@ def exists(package: str): return True except ImportError: return False - - -class _LazyModule(ModuleType): - """ - Module class that surfaces all objects but only performs associated imports when the objects are requested. - """ - - # Very heavily inspired by optuna.integration._IntegrationModule - # https://github.com/optuna/optuna/blob/master/optuna/integration/__init__.py - def __init__(self, name, module_file, import_structure, module_spec=None, extra_objects=None): - super().__init__(name) - self._modules = set(import_structure.keys()) - self._class_to_module = {} - for key, values in import_structure.items(): - for value in values: - self._class_to_module[value] = key - # Needed for autocompletion in an IDE - self.__all__ = list(import_structure.keys()) + list(chain(*import_structure.values())) - self.__file__ = module_file - self.__spec__ = module_spec - self.__path__ = [os.path.dirname(module_file)] - self._objects = {} if extra_objects is None else extra_objects - self._name = name - self._import_structure = import_structure - - # Needed for autocompletion in an IDE - def __dir__(self): - result = super().__dir__() - # The elements of self.__all__ that are submodules may or may not be in the dir already, depending on whether - # they have been accessed or not. So we only add the elements of self.__all__ that are not already in the dir. - for attr in self.__all__: - if attr not in result: - result.append(attr) - return result - - def __getattr__(self, name: str) -> Any: - if name in self._objects: - return self._objects[name] - if name in self._modules: - value = self._get_module(name) - elif name in self._class_to_module.keys(): - module = self._get_module(self._class_to_module[name]) - value = getattr(module, name) - else: - raise AttributeError(f'module {self.__name__} has no attribute {name}') - - setattr(self, name, value) - return value - - def _get_module(self, module_name: str): - return importlib.import_module('.' + module_name, self.__name__) - - def __reduce__(self): - return self.__class__, (self._name, self.__file__, self._import_structure) diff --git a/src/twinkle_client/__init__.py b/src/twinkle_client/__init__.py index 054f073c..a3614d75 100644 --- a/src/twinkle_client/__init__.py +++ b/src/twinkle_client/__init__.py @@ -1,9 +1,11 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from __future__ import annotations -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from .data_plane import DataPlaneClient + from .http import ClientContext, ClientTransport from .manager import TwinkleClient @@ -73,7 +75,17 @@ def init_twinkle_client( ) -from .data_plane import DataPlaneClient # noqa: E402 (late import: keeps optional data-plane dep off the hot path) -from .http import ClientContext, ClientTransport # noqa: E402 +def __getattr__(name: str) -> Any: + if name == 'DataPlaneClient': + from .data_plane import DataPlaneClient + value = DataPlaneClient + elif name in {'ClientContext', 'ClientTransport'}: + from .http import ClientContext, ClientTransport + value = {'ClientContext': ClientContext, 'ClientTransport': ClientTransport}[name] + else: + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + globals()[name] = value + return value + __all__ = ['ClientContext', 'ClientTransport', 'DataPlaneClient', 'init_tinker_client', 'init_twinkle_client'] diff --git a/src/twinkle_client/_future.py b/src/twinkle_client/_future.py index 18237487..b8fc5a3b 100644 --- a/src/twinkle_client/_future.py +++ b/src/twinkle_client/_future.py @@ -12,10 +12,10 @@ import time from typing import Any, Optional +from twinkle.protocol.types.lifecycle import TERMINAL_STATUSES, TaskEnvelope from twinkle_client.exceptions import TaskCancelledError, TaskFailedError, TaskRecordLostError, TaskWaitTimeoutError from twinkle_client.http import ClientTransport from twinkle_client.http.context import capture_transport -from twinkle_client.types.lifecycle import TERMINAL_STATUSES, TaskEnvelope logger = logging.getLogger('twinkle_client') @@ -35,11 +35,11 @@ def _retrieve_url(transport: ClientTransport) -> str: - return f'{transport.context.base_url}/twinkle/retrieve_future' + return transport.url('twinkle/retrieve_future') def _cancel_url(transport: ClientTransport) -> str: - return f'{transport.context.base_url}/twinkle/cancel' + return transport.url('twinkle/cancel') def _best_effort_cancel(request_id: str, transport: ClientTransport) -> None: @@ -140,6 +140,12 @@ def resolve( try: reply = _post_retrieve(submit.request_id, resolved_transport) transport_failures = not_found_count = 0 + except (requests.ConnectionError, requests.Timeout): + transport_failures += 1 + if transport_failures > _TRANSPORT_RETRY_MAX: + raise + time.sleep(min(2**transport_failures, 30)) + continue except requests.HTTPError as e: status = _status_of(e) if status == 404: diff --git a/src/twinkle_client/_request_builder.py b/src/twinkle_client/_request_builder.py index 75c1399c..3afc8b5d 100644 --- a/src/twinkle_client/_request_builder.py +++ b/src/twinkle_client/_request_builder.py @@ -21,38 +21,42 @@ """ from __future__ import annotations +from dataclasses import asdict, is_dataclass from pydantic import BaseModel from typing import Any, Mapping -from twinkle_client.common.json_utils import json_safe +from twinkle.protocol.json_utils import json_safe +from twinkle.protocol.types.base import FieldRole, fields_with_role from twinkle_client.exceptions import TwinkleClientValidationError -from twinkle_client.types.base import FieldRole, fields_with_role def to_wire_value(value: Any) -> Any: - """Convert one caller-supplied argument to a JSON-native value. + """Recursively convert a caller value to its JSON wire representation. - Handles the three object kinds the client has always accepted in a request body: - a server-side component handle (sent as its id), a ``DatasetMeta`` / ``LoraConfig`` - (sent as the canonical serialized form the server decodes), and numpy / torch - values (sent as nested lists). - - Anything else is passed through for the model to validate, so an unsupported type - is reported by pydantic with its field path instead of by a generic error here. + ``ClientTransport.post`` and schema-driven request construction both use this + function. ``post_model`` remains a separate Pydantic JSON entry point and + deliberately excludes unset optionals. """ - # A remote-component handle (InputProcessor / dataset / dataloader wrapper) - # is sent as its server-side id. Guarded on ``str`` so an unrelated object - # that merely happens to expose a ``processor_id`` attribute is not silently - # coerced to something that is not an id. + if isinstance(value, (str, int, float, bool, type(None))): + return value + if isinstance(value, (bytes, bytearray, memoryview)): + raise TwinkleClientValidationError('Binary values are not supported by the JSON transport') component_id = getattr(value, 'processor_id', None) if isinstance(component_id, str): return component_id + if isinstance(value, Mapping): + return {str(key): to_wire_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set, frozenset)): + return [to_wire_value(item) for item in value] + from peft import LoraConfig from twinkle.dataset import DatasetMeta if isinstance(value, (DatasetMeta, LoraConfig)): - from twinkle_client.common.serialize import serialize_object + from twinkle.protocol.serialize import serialize_object return serialize_object(value) + if is_dataclass(value) and not isinstance(value, type): + return to_wire_value(asdict(value)) if isinstance(value, BaseModel): return value.model_dump(mode='json') return json_safe(value) diff --git a/src/twinkle_client/common/__init__.py b/src/twinkle_client/common/__init__.py index 930ef1ba..ec8001aa 100644 --- a/src/twinkle_client/common/__init__.py +++ b/src/twinkle_client/common/__init__.py @@ -1,7 +1,7 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Client-internal helpers shared across the twinkle_client subpackages. -Regular package (carries this ``__init__``) so the modules here -- ``component_rpc``, -``serialize``, ``json_utils`` -- are included by ``setuptools.packages.find`` in a -built wheel; a namespace-only directory would be dropped from the distribution. +Regular package (carries this ``__init__``) so ``component_rpc`` is included by +``setuptools.packages.find`` in a built wheel; a namespace-only directory would +be dropped from the distribution. """ diff --git a/src/twinkle_client/common/component_rpc.py b/src/twinkle_client/common/component_rpc.py index c7b631bb..65b87745 100644 --- a/src/twinkle_client/common/component_rpc.py +++ b/src/twinkle_client/common/component_rpc.py @@ -4,17 +4,12 @@ from typing import Any +from twinkle.protocol.types.processor import (ProcessorCallRequest, ProcessorCallResponse, ProcessorCreateRequest, + ProcessorCreateResponse) from twinkle_client._request_builder import build_request from twinkle_client.http import ClientTransport +from twinkle_client.http.client import DEFAULT_TIMEOUT from twinkle_client.http.context import capture_transport -from twinkle_client.types.processor import (ProcessorCallRequest, ProcessorCallResponse, ProcessorCreateRequest, - ProcessorCreateResponse) - -_UNSET = object() - - -def processor_base_url(transport: ClientTransport) -> str: - return f'{transport.context.base_url}/processor/twinkle' def create_remote_component( @@ -27,14 +22,14 @@ def create_remote_component( """Create a server-side component using one captured transport.""" resolved = capture_transport(transport) body = build_request(ProcessorCreateRequest, processor_type=processor_type, class_type=class_type, **init_kwargs) - response = resolved.post_model(f'{processor_base_url(resolved)}/create', body) + response = resolved.post_model(resolved.url('processor/twinkle/create'), body) return ProcessorCreateResponse(**response.json()).processor_id def call_remote_component( processor_id: str, function: str, - http_timeout: Any = _UNSET, + http_timeout: Any = DEFAULT_TIMEOUT, /, *, transport: ClientTransport | None = None, @@ -43,8 +38,5 @@ def call_remote_component( """Invoke one server-side component using its owner's transport.""" resolved = capture_transport(transport) body = build_request(ProcessorCallRequest, processor_id=processor_id, function=function, **call_kwargs) - url = f'{processor_base_url(resolved)}/call' - response = ( - resolved.post_model(url, body) if http_timeout is _UNSET else resolved.post_model( - url, body, timeout=http_timeout)) + response = resolved.post_model(resolved.url('processor/twinkle/call'), body, timeout=http_timeout) return ProcessorCallResponse(**response.json()).result diff --git a/src/twinkle_client/common/remote_component.py b/src/twinkle_client/common/remote_component.py new file mode 100644 index 00000000..a5017f78 --- /dev/null +++ b/src/twinkle_client/common/remote_component.py @@ -0,0 +1,45 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Shared transport binding for processor-backed remote component wrappers.""" +from __future__ import annotations + +from typing import Any + +from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.http import ClientTransport +from twinkle_client.http.context import capture_transport + + +class RemoteComponent: + """Bind one remote component id to one captured transport. + + The class intentionally has no ``__init__``: concrete wrappers retain + ownership of their domain-specific constructor and MRO. + """ + + _transport: ClientTransport + processor_id: str + + def _bind_remote( + self, + processor_type: str, + class_type: str, + *, + transport: ClientTransport | None = None, + **kwargs: Any, + ) -> None: + self._transport = capture_transport(transport) + self.processor_id = create_remote_component( + processor_type, + class_type, + transport=self._transport, + **kwargs, + ) + + def _call(self, function: str, *args: Any, **kwargs: Any) -> Any: + return call_remote_component( + self.processor_id, + function, + *args, + transport=self._transport, + **kwargs, + ) diff --git a/src/twinkle_client/data_plane.py b/src/twinkle_client/data_plane.py index 8fe9a217..eec09716 100644 --- a/src/twinkle_client/data_plane.py +++ b/src/twinkle_client/data_plane.py @@ -6,10 +6,10 @@ from collections.abc import Callable from typing import Any, TypeVar -from twinkle_client.common.json_utils import json_safe +from twinkle.protocol.json_utils import json_safe +from twinkle.protocol.types.component import DataRef, DataRowsResponse from twinkle_client.http import ClientTransport from twinkle_client.http.context import capture_transport -from twinkle_client.types.component import DataRef, DataRowsResponse _T = TypeVar('_T') @@ -23,7 +23,7 @@ class DataPlaneClient: def __init__(self, server_url: str | None = None, *, transport: ClientTransport | None = None): self._transport = capture_transport(transport) - self.server_url = (server_url or f'{self._transport.context.base_url}/data-plane').rstrip('/') + self.server_url = (server_url or self._transport.url('data-plane')).rstrip('/') def put( self, diff --git a/src/twinkle_client/dataloader/dataloader.py b/src/twinkle_client/dataloader/dataloader.py index 376b7726..21cd5cdb 100644 --- a/src/twinkle_client/dataloader/dataloader.py +++ b/src/twinkle_client/dataloader/dataloader.py @@ -1,19 +1,22 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -from typing import Callable, Type, Union +from __future__ import annotations -from twinkle.dataset import Dataset -from twinkle.processor import InputProcessor -from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from typing import TYPE_CHECKING, Callable, Type, Union + +from twinkle_client.common.remote_component import RemoteComponent from twinkle_client.http import ClientTransport -from twinkle_client.http.context import capture_transport + +if TYPE_CHECKING: + from twinkle.processor import InputProcessor + from twinkle_client.dataset import Dataset -class DataLoader: +class DataLoader(RemoteComponent): """Client wrapper for DataLoader that calls server HTTP endpoints.""" def __init__( self, - dataset: Union[Dataset, Callable], + dataset: Dataset | Callable, *, transport: ClientTransport | None = None, **kwargs, @@ -21,17 +24,13 @@ def __init__( dataset_transport = getattr(dataset, '_transport', None) if transport is not None and dataset_transport is not None and transport is not dataset_transport: raise ValueError('DataLoader and its remote Dataset must use the same ClientTransport') - self._transport = capture_transport(transport or dataset_transport) - self.processor_id = create_remote_component( - 'dataloader', 'DataLoader', dataset=dataset, transport=self._transport, **kwargs) - - def _call(self, function: str, *args, **kwargs): - return call_remote_component(self.processor_id, function, *args, transport=self._transport, **kwargs) + self._bind_remote( + 'dataloader', 'DataLoader', dataset=dataset, transport=transport or dataset_transport, **kwargs) def __len__(self): return self._call('__len__') - def set_processor(self, processor_cls: Union[Type[InputProcessor], str, InputProcessor, Callable], **kwargs): + def set_processor(self, processor_cls: type[InputProcessor] | str | InputProcessor | Callable, **kwargs): return self._call('set_processor', processor_cls=processor_cls, **kwargs) def __iter__(self): diff --git a/src/twinkle_client/dataset/base.py b/src/twinkle_client/dataset/base.py index f22fb0d8..c7a7b8a9 100644 --- a/src/twinkle_client/dataset/base.py +++ b/src/twinkle_client/dataset/base.py @@ -4,12 +4,11 @@ from twinkle.dataset import DatasetMeta from twinkle.preprocessor import DataFilter, Preprocessor from twinkle.template import Template -from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.common.remote_component import RemoteComponent from twinkle_client.http import ClientTransport -from twinkle_client.http.context import capture_transport -class Dataset: +class Dataset(RemoteComponent): """Client wrapper for Dataset that calls server HTTP endpoints.""" def __init__( @@ -19,12 +18,7 @@ def __init__( transport: ClientTransport | None = None, **kwargs, ): - self._transport = capture_transport(transport) - self.processor_id = create_remote_component( - 'dataset', 'Dataset', dataset_meta=dataset_meta, transport=self._transport, **kwargs) - - def _call(self, function: str, *args, **kwargs): - return call_remote_component(self.processor_id, function, *args, transport=self._transport, **kwargs) + self._bind_remote('dataset', 'Dataset', dataset_meta=dataset_meta, transport=transport, **kwargs) def set_template(self, template_func: Union[Template, Type[Template], str], **kwargs): return self._call('set_template', template_func=template_func, **kwargs) diff --git a/src/twinkle_client/dataset/iterable_dataset.py b/src/twinkle_client/dataset/iterable_dataset.py index a7ad36e9..d3b5eece 100644 --- a/src/twinkle_client/dataset/iterable_dataset.py +++ b/src/twinkle_client/dataset/iterable_dataset.py @@ -2,13 +2,16 @@ from torch.utils.data import IterableDataset as TorchIterableDataset from twinkle.dataset import DatasetMeta -from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.common.remote_component import RemoteComponent from twinkle_client.http import ClientTransport -from twinkle_client.http.context import capture_transport -class IterableDataset(TorchIterableDataset): - """Client wrapper for IterableDataset that calls server HTTP endpoints.""" +class IterableDataset(TorchIterableDataset, RemoteComponent): + """Remote iterable backed by one server-side cursor. + + Iteration is stateful and does not support concurrent or repeated iteration + over the same wrapper instance. + """ def __init__( self, @@ -17,22 +20,11 @@ def __init__( transport: ClientTransport | None = None, **kwargs, ): - self._transport = capture_transport(transport) - self.processor_id = create_remote_component( - 'dataset', 'IterableDataset', dataset_meta=dataset_meta, transport=self._transport, **kwargs) - - def _call(self, function: str, *args, **kwargs): - return call_remote_component(self.processor_id, function, *args, transport=self._transport, **kwargs) + self._bind_remote('dataset', 'IterableDataset', dataset_meta=dataset_meta, transport=transport, **kwargs) def add_dataset(self, dataset_meta: DatasetMeta, **kwargs): return self._call('add_dataset', dataset_meta=dataset_meta, **kwargs) - def __len__(self): - return self._call('__len__') - - def __getitem__(self, idx): - return self._call('__getitem__', idx=idx) - def __iter__(self): self._call('__iter__') return self diff --git a/src/twinkle_client/dataset/iterable_packing_dataset.py b/src/twinkle_client/dataset/iterable_packing_dataset.py index 42991a6d..5fc91174 100644 --- a/src/twinkle_client/dataset/iterable_packing_dataset.py +++ b/src/twinkle_client/dataset/iterable_packing_dataset.py @@ -1,16 +1,15 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -from torch.utils.data import IterableDataset +from torch.utils.data import IterableDataset as TorchIterableDataset from typing import Type, Union from twinkle.dataset import DatasetMeta from twinkle.template import Template -from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.common.remote_component import RemoteComponent from twinkle_client.http import ClientTransport -from twinkle_client.http.context import capture_transport -class IterablePackingDataset(IterableDataset): - """Client wrapper for IterablePackingDataset that calls server HTTP endpoints.""" +class IterablePackingDataset(TorchIterableDataset, RemoteComponent): + """Remote packing iterable backed by one non-reentrant server cursor.""" def __init__( self, @@ -22,21 +21,17 @@ def __init__( transport: ClientTransport | None = None, **kwargs, ): - self._transport = capture_transport(transport) - self.processor_id = create_remote_component( + self._bind_remote( 'dataset', 'IterablePackingDataset', dataset_meta=dataset_meta, packing_interval=packing_interval, packing_num_proc=packing_num_proc, cyclic=cyclic, - transport=self._transport, + transport=transport, **kwargs, ) - def _call(self, function: str, *args, **kwargs): - return call_remote_component(self.processor_id, function, *args, transport=self._transport, **kwargs) - def set_template(self, template_cls: Union[Type[Template], str, Template], **kwargs): return self._call('set_template', template_cls=template_cls, **kwargs) diff --git a/src/twinkle_client/dataset/lazy_dataset.py b/src/twinkle_client/dataset/lazy_dataset.py index 18f95857..866fd64e 100644 --- a/src/twinkle_client/dataset/lazy_dataset.py +++ b/src/twinkle_client/dataset/lazy_dataset.py @@ -1,11 +1,6 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -from typing import Any, Callable, Dict, Optional, Type, Union - from twinkle.dataset import DatasetMeta -from twinkle.preprocessor import DataFilter, Preprocessor -from twinkle_client.common.component_rpc import create_remote_component from twinkle_client.http import ClientTransport -from twinkle_client.http.context import capture_transport from .base import Dataset @@ -19,39 +14,4 @@ def __init__( transport: ClientTransport | None = None, **kwargs, ): - self._transport = capture_transport(transport) - self.processor_id = create_remote_component( - 'dataset', 'LazyDataset', dataset_meta=dataset_meta, transport=self._transport, **kwargs) - - def map(self, - preprocess_func: Union[Preprocessor, Callable, str, Type[Preprocessor]], - dataset_meta: DatasetMeta = None, - init_args: Dict[str, Any] = None, - **kwargs): - return self._call( - 'map', preprocess_func=preprocess_func, dataset_meta=dataset_meta, init_args=init_args, **kwargs) - - def filter(self, - filter_func: Union[Callable, str, Type[DataFilter], DataFilter], - dataset_meta: DatasetMeta = None, - init_args: Dict[str, Any] = None, - **kwargs): - return self._call('filter', filter_func=filter_func, dataset_meta=dataset_meta, init_args=init_args, **kwargs) - - def add_dataset(self, dataset_meta: DatasetMeta, **kwargs): - return self._call('add_dataset', dataset_meta=dataset_meta, **kwargs) - - def mix_dataset(self, interleave=True): - return self._call('mix_dataset', interleave=interleave) - - def encode(self, add_generation_prompt: bool = False, timeout: Optional[int] = 600, **kwargs): - return self._call('encode', timeout, add_generation_prompt=add_generation_prompt, **kwargs) - - def check(self, **kwargs): - return self._call('check', **kwargs) - - def __getitem__(self, idx): - return self._call('__getitem__', idx=idx) - - def __len__(self): - return self._call('__len__') + self._bind_remote('dataset', 'LazyDataset', dataset_meta=dataset_meta, transport=transport, **kwargs) diff --git a/src/twinkle_client/dataset/packing_dataset.py b/src/twinkle_client/dataset/packing_dataset.py index b17185e0..bf5c014e 100644 --- a/src/twinkle_client/dataset/packing_dataset.py +++ b/src/twinkle_client/dataset/packing_dataset.py @@ -1,8 +1,6 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from twinkle.dataset import DatasetMeta -from twinkle_client.common.component_rpc import create_remote_component from twinkle_client.http import ClientTransport -from twinkle_client.http.context import capture_transport from .base import Dataset @@ -17,13 +15,12 @@ def __init__( transport: ClientTransport | None = None, **kwargs, ): - self._transport = capture_transport(transport) - self.processor_id = create_remote_component( + self._bind_remote( 'dataset', 'PackingDataset', dataset_meta=dataset_meta, packing_num_proc=packing_num_proc, - transport=self._transport, + transport=transport, **kwargs, ) diff --git a/src/twinkle_client/exceptions.py b/src/twinkle_client/exceptions.py index e9a79c63..dd56d6e3 100644 --- a/src/twinkle_client/exceptions.py +++ b/src/twinkle_client/exceptions.py @@ -18,7 +18,7 @@ import requests from typing import Any, Optional -from twinkle_client.types.errors import ErrorCategory +from twinkle.protocol.types.errors import ErrorCategory class TwinkleClientValidationError(ValueError): diff --git a/src/twinkle_client/http/__init__.py b/src/twinkle_client/http/__init__.py index 55d93e97..700e46ac 100644 --- a/src/twinkle_client/http/__init__.py +++ b/src/twinkle_client/http/__init__.py @@ -1,12 +1,8 @@ """Public HTTP transport API.""" -from .client import ClientTransport, http_delete, http_get, http_post, http_post_model +from .client import ClientTransport from .context import ClientContext __all__ = [ 'ClientContext', 'ClientTransport', - 'http_get', - 'http_post', - 'http_post_model', - 'http_delete', ] diff --git a/src/twinkle_client/http/client.py b/src/twinkle_client/http/client.py index 77719c78..a209bc09 100644 --- a/src/twinkle_client/http/client.py +++ b/src/twinkle_client/http/client.py @@ -1,40 +1,22 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Instance-owned HTTP transport and legacy module-level compatibility facade.""" +"""Instance-owned HTTP transport.""" from __future__ import annotations import requests from collections.abc import Mapping -from dataclasses import asdict, is_dataclass +from requests.adapters import HTTPAdapter from typing import Any +from urllib3.util.retry import Retry +from twinkle.protocol.headers import build_routing_headers +from twinkle.protocol.types.errors import ErrorCategory, ErrorPayload +from twinkle_client._request_builder import to_wire_value from twinkle_client.exceptions import TwinkleClientValidationError, TwinkleHTTPError -from twinkle_client.types.errors import ErrorCategory, ErrorPayload from .context import ClientContext, capture_transport -from .headers import build_routing_headers # Must be greater than the server long-poll window and below common gateway idle limits. _HTTP_TIMEOUT = 90 -_UNSET = object() -_JSON_PRIMITIVES = (str, int, float, bool, type(None)) - - -def _serialize_value(value: Any) -> Any: - if isinstance(value, _JSON_PRIMITIVES): - return value - if isinstance(value, bytes | bytearray | memoryview): - raise TwinkleClientValidationError('Binary values are not supported by the JSON transport') - if isinstance(value, Mapping): - return {str(key): _serialize_value(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_serialize_value(item) for item in value] - if is_dataclass(value) and not isinstance(value, type): - return _serialize_value(asdict(value)) - # Single source of truth for leaf/domain objects (pydantic models, remote - # component handles, DatasetMeta / LoraConfig, numpy / torch): reuse the - # request builder's converter so a value serializes identically whether it - # goes out via ``post(json_data=...)`` or via ``post_model(body=...)``. - from twinkle_client._request_builder import to_wire_value - return to_wire_value(value) +DEFAULT_TIMEOUT = object() def _handle_response(response: requests.Response) -> requests.Response: @@ -89,7 +71,18 @@ def _handle_response(response: requests.Response) -> requests.Response: class ClientTransport: - """The sole request-time owner of URL, identity, headers, and HTTP resources.""" + """The sole request-time owner of URL, identity, headers, and HTTP resources. + + ``post`` recursively converts arbitrary JSON-like values through + :func:`to_wire_value`; ``post_model`` serializes a validated Pydantic model + directly and excludes ``None`` fields. The two entry points intentionally + remain distinct. + + The adapter retries only idempotent GET/DELETE requests. POST is never + transparently replayed because control-plane calls such as ``create_session`` + do not carry a deduplication key; read-only future retrieval handles retries + explicitly in the future layer. + """ def __init__( self, @@ -97,11 +90,35 @@ def __init__( *, session: requests.Session | None = None, timeout: float = _HTTP_TIMEOUT, + pool_maxsize: int = 32, ) -> None: + if pool_maxsize < 2: + raise ValueError('pool_maxsize must be at least 2') self._context = context self._session = session or requests.Session() + if session is None: + retry = Retry( + total=3, + connect=3, + read=0, + status=3, + backoff_factor=0.25, + status_forcelist=(408, 429, 500, 502, 503, 504), + allowed_methods=frozenset({'GET', 'DELETE'}), + respect_retry_after_header=True, + raise_on_status=False, + ) + adapter = HTTPAdapter( + max_retries=retry, + pool_connections=pool_maxsize, + pool_maxsize=pool_maxsize, + pool_block=True, + ) + self._session.mount('http://', adapter) + self._session.mount('https://', adapter) self._timeout = timeout self._closed = False + self._published = False self._capabilities: object | None = None @property @@ -115,8 +132,13 @@ def closed(self) -> bool: def bind_context(self, context: ClientContext) -> None: """Replace provisional identity before the transport is published to wrappers.""" self._ensure_open() + if self._published: + raise RuntimeError('Cannot rebind a published ClientTransport') self._context = context + def _mark_published(self) -> None: + self._published = True + @property def cached_capabilities(self) -> object | None: return self._capabilities @@ -141,58 +163,52 @@ def _headers(self, additional_headers: Mapping[str, str] | None = None) -> dict[ return headers def _request_timeout(self, timeout: object) -> float | None: - return self._timeout if timeout is _UNSET else timeout # type: ignore[return-value] + return self._timeout if timeout is DEFAULT_TIMEOUT else timeout # type: ignore[return-value] def _ensure_open(self) -> None: if self._closed: raise RuntimeError('ClientTransport is closed') - def get( - self, - path_or_url: str = '', - *, - params: Mapping[str, Any] | None = None, - headers: Mapping[str, str] | None = None, - timeout: float | None | object = _UNSET, - ) -> requests.Response: + def get(self, + path_or_url: str = '', + *, + params: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + timeout: float | None | object = DEFAULT_TIMEOUT) -> requests.Response: self._ensure_open() response = self._session.get( self.url(path_or_url), headers=self._headers(headers), - params=_serialize_value(params or {}), + params=to_wire_value(params or {}), timeout=self._request_timeout(timeout), ) return _handle_response(response) - def post( - self, - path_or_url: str = '', - *, - json_data: Mapping[str, Any] | None = None, - data: Any = None, - headers: Mapping[str, str] | None = None, - timeout: float | None | object = _UNSET, - ) -> requests.Response: + def post(self, + path_or_url: str = '', + *, + json_data: Mapping[str, Any] | None = None, + data: Any = None, + headers: Mapping[str, str] | None = None, + timeout: float | None | object = DEFAULT_TIMEOUT) -> requests.Response: self._ensure_open() if isinstance(data, (bytes, bytearray, memoryview)): raise TwinkleClientValidationError('Binary request bodies are not supported by this transport') response = self._session.post( self.url(path_or_url), headers=self._headers(headers), - json=_serialize_value(json_data or {}), + json=to_wire_value(json_data or {}), data=data, timeout=self._request_timeout(timeout), ) return _handle_response(response) - def post_model( - self, - path_or_url: str, - body: Any, - *, - headers: Mapping[str, str] | None = None, - timeout: float | None | object = _UNSET, - ) -> requests.Response: + def post_model(self, + path_or_url: str, + body: Any, + *, + headers: Mapping[str, str] | None = None, + timeout: float | None | object = DEFAULT_TIMEOUT) -> requests.Response: from twinkle_client._request_builder import request_json self._ensure_open() request_headers = {'content-type': 'application/json', **dict(headers or {})} @@ -204,19 +220,17 @@ def post_model( ) return _handle_response(response) - def delete( - self, - path_or_url: str = '', - *, - params: Mapping[str, Any] | None = None, - headers: Mapping[str, str] | None = None, - timeout: float | None | object = _UNSET, - ) -> requests.Response: + def delete(self, + path_or_url: str = '', + *, + params: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + timeout: float | None | object = DEFAULT_TIMEOUT) -> requests.Response: self._ensure_open() response = self._session.delete( self.url(path_or_url), headers=self._headers(headers), - params=_serialize_value(params or {}), + params=to_wire_value(params or {}), timeout=self._request_timeout(timeout), ) return _handle_response(response) @@ -226,51 +240,3 @@ def close(self) -> None: return self._closed = True self._session.close() - - -# Compatibility facade. Core wrappers always pass their captured transport explicitly; -# only legacy external callers may omit it and resolve the current default here. -def http_get( - url: str | None = None, - params: Mapping[str, Any] | None = None, - additional_headers: Mapping[str, str] | None = None, - timeout: float | None = _HTTP_TIMEOUT, - *, - transport: ClientTransport | None = None, -) -> requests.Response: - return capture_transport(transport).get(url or '', params=params, headers=additional_headers, timeout=timeout) - - -def http_post( - url: str | None = None, - json_data: Mapping[str, Any] | None = None, - data: Any = None, - additional_headers: Mapping[str, str] | None = None, - timeout: float | None = _HTTP_TIMEOUT, - *, - transport: ClientTransport | None = None, -) -> requests.Response: - return capture_transport(transport).post( - url or '', json_data=json_data, data=data, headers=additional_headers, timeout=timeout) - - -def http_post_model( - url: str, - body: Any, - additional_headers: Mapping[str, str] | None = None, - timeout: float | None = _HTTP_TIMEOUT, - *, - transport: ClientTransport | None = None, -) -> requests.Response: - return capture_transport(transport).post_model(url, body, headers=additional_headers, timeout=timeout) - - -def http_delete( - url: str | None = None, - params: Mapping[str, Any] | None = None, - additional_headers: Mapping[str, str] | None = None, - timeout: float | None = _HTTP_TIMEOUT, - *, - transport: ClientTransport | None = None, -) -> requests.Response: - return capture_transport(transport).delete(url or '', params=params, headers=additional_headers, timeout=timeout) diff --git a/src/twinkle_client/http/context.py b/src/twinkle_client/http/context.py index 858fa978..58bb8d29 100644 --- a/src/twinkle_client/http/context.py +++ b/src/twinkle_client/http/context.py @@ -2,6 +2,7 @@ """Immutable client identity and the compatibility default-transport registry.""" from __future__ import annotations +import logging import os import threading import uuid @@ -14,6 +15,8 @@ TWINKLE_SERVER_URL = os.environ.get('TWINKLE_SERVER_URL', 'http://127.0.0.1:8000') TWINKLE_SERVER_TOKEN = os.environ.get('TWINKLE_SERVER_TOKEN', 'EMPTY_TOKEN') +logger = logging.getLogger('twinkle_client') + def _normalize_base_url(base_url: str) -> str: base_url = base_url.rstrip('/') @@ -54,10 +57,13 @@ def capture_transport(explicit: ClientTransport | None = None) -> ClientTranspor if explicit is not None: if explicit.closed: raise RuntimeError('Cannot capture a closed ClientTransport') + explicit._mark_published() return explicit with _default_lock: if _default_transport is None or _default_transport.closed: _default_transport = _new_env_transport() + logger.info('No explicit Twinkle client configured; using %s', _default_transport.context.base_url) + _default_transport._mark_published() return _default_transport @@ -66,6 +72,13 @@ def set_default_transport(transport: ClientTransport) -> None: raise RuntimeError('Cannot register a closed ClientTransport') global _default_transport with _default_lock: + if _default_transport is not None and _default_transport is not transport and not _default_transport.closed: + logger.warning( + 'Replacing default Twinkle transport %s with %s; existing wrappers retain the old transport', + _default_transport.context.base_url, + transport.context.base_url, + ) + transport._mark_published() _default_transport = transport diff --git a/src/twinkle_client/manager.py b/src/twinkle_client/manager.py index 92effd5b..e87ff229 100644 --- a/src/twinkle_client/manager.py +++ b/src/twinkle_client/manager.py @@ -8,20 +8,17 @@ from typing import Any from twinkle import get_logger +from twinkle.protocol.types.server import CapacityInfoResponse, DeleteCheckpointResponse, GetServerCapabilitiesResponse +from twinkle.protocol.types.session import CreateSessionRequest, CreateSessionResponse, SessionHeartbeatRequest +from twinkle.protocol.types.training import (Checkpoint, Cursor, ParsedCheckpointTwinklePath, TrainingRun, + WeightsInfoResponse) from twinkle_client.exceptions import TwinkleHTTPError from twinkle_client.http import ClientContext, ClientTransport from twinkle_client.http.context import (TWINKLE_SERVER_TOKEN, TWINKLE_SERVER_URL, clear_default_transport, set_default_transport) -from twinkle_client.types.server import CapacityInfoResponse, DeleteCheckpointResponse, GetServerCapabilitiesResponse -from twinkle_client.types.session import CreateSessionRequest, CreateSessionResponse, SessionHeartbeatRequest -from twinkle_client.types.training import (Checkpoint, Cursor, ParsedCheckpointTwinklePath, TrainingRun, - WeightsInfoResponse) logger = get_logger() -# Compatibility import: HTTP failures now have one canonical implementation. -TwinkleClientError = TwinkleHTTPError - class TwinkleClient: """Owner of one connected transport, remote session, and heartbeat thread. @@ -35,11 +32,13 @@ def __init__( self, *, transport: ClientTransport, + heartbeat_transport: ClientTransport | None = None, route_prefix: str = '/twinkle', session_heartbeat_interval: int = 10, ) -> None: """Build an already-connected client without performing remote I/O.""" self._transport = transport + self._heartbeat_transport = heartbeat_transport or transport self.base_url = transport.context.base_url self.api_key = transport.context.api_key self.route_prefix = route_prefix.rstrip('/') if route_prefix else '' @@ -67,6 +66,7 @@ def connect( transport = ClientTransport(context) prefix = route_prefix.rstrip('/') if route_prefix else '' client = None + heartbeat_transport = None try: response = transport.post( f'{context.base_url}{prefix}/create_session', @@ -74,8 +74,10 @@ def connect( ) session_id = CreateSessionResponse.model_validate(response.json()).session_id transport.bind_context(replace(context, session_id=session_id)) + heartbeat_transport = ClientTransport(transport.context) client = cls( transport=transport, + heartbeat_transport=heartbeat_transport, route_prefix=prefix, session_heartbeat_interval=session_heartbeat_interval, ) @@ -85,6 +87,8 @@ def connect( return client except BaseException: if client is None: + if heartbeat_transport is not None: + heartbeat_transport.close() transport.close() else: client.close() @@ -107,11 +111,11 @@ def get_capacity_info(self) -> CapacityInfoResponse: Get the server's global LoRA capacity information. Returns: - :class:`~twinkle_client.types.server.CapacityInfoResponse` with + :class:`~twinkle.protocol.types.server.CapacityInfoResponse` with ``max_loras``, ``used_loras``, and ``free_loras`` fields. Raises: - TwinkleClientError: If the request fails. + TwinkleHTTPError: If the request fails. """ response = self._transport.get(self._get_url('/capacity_info')) data = response.json() @@ -136,7 +140,7 @@ def create_session(self, metadata: dict[str, Any] | None = None) -> str: The session ID string. Raises: - TwinkleClientError: If the session creation request fails. + TwinkleHTTPError: If the session creation request fails. """ resp = self._transport.post( self._get_url('/create_session'), @@ -158,7 +162,7 @@ def _touch_session_loop(self) -> None: success = False try: logger.debug(f'[TwinkleClient] Touching session (session={self._session_id})...') - self._transport.post( + self._heartbeat_transport.post( self._get_url('/session_heartbeat'), json_data=SessionHeartbeatRequest(session_id=self._session_id).model_dump(), timeout=min(self._heartbeat_interval, 10), @@ -182,6 +186,8 @@ def close(self) -> None: if self._heartbeat_thread is not None and self._heartbeat_thread.is_alive(): self._heartbeat_thread.join(timeout=max(2, min(self._heartbeat_interval, 10))) clear_default_transport(self._transport) + if self._heartbeat_transport is not self._transport: + self._heartbeat_transport.close() self._transport.close() try: atexit.unregister(self.close) @@ -226,11 +232,11 @@ def get_server_capabilities(self) -> GetServerCapabilitiesResponse: Get the server's supported models and capabilities. Returns: - :class:`~twinkle_client.types.server.GetServerCapabilitiesResponse` with + :class:`~twinkle.protocol.types.server.GetServerCapabilitiesResponse` with ``supported_models`` field containing a list of supported model names. Raises: - TwinkleClientError: If the request fails. + TwinkleHTTPError: If the request fails. """ cached = self._transport.cached_capabilities if isinstance(cached, GetServerCapabilitiesResponse): @@ -256,10 +262,10 @@ def list_training_runs(self, limit: int = 20, offset: int = 0, all_users: bool = all_users: If True, return all runs (if permission allows). Returns: - List of :class:`~twinkle_client.types.training.TrainingRun` objects. + List of :class:`~twinkle.protocol.types.training.TrainingRun` objects. Raises: - TwinkleClientError: If the request fails. + TwinkleHTTPError: If the request fails. """ params: dict[str, Any] = {'limit': limit, 'offset': offset} if all_users: @@ -288,7 +294,7 @@ def list_training_runs_with_cursor( Tuple of (list of TrainingRun, Cursor with pagination info). Raises: - TwinkleClientError: If the request fails. + TwinkleHTTPError: If the request fails. """ params: dict[str, Any] = {'limit': limit, 'offset': offset} if all_users: @@ -309,10 +315,10 @@ def get_training_run(self, run_id: str) -> TrainingRun: run_id: The training run identifier. Returns: - :class:`~twinkle_client.types.training.TrainingRun` object with run details. + :class:`~twinkle.protocol.types.training.TrainingRun` object with run details. Raises: - TwinkleClientError: If run not found or access denied. + TwinkleHTTPError: If run not found or access denied. """ response = self._transport.get(self._get_url(f'/training_runs/{run_id}')) data = response.json() @@ -330,10 +336,10 @@ def list_checkpoints(self, run_id: str) -> list[Checkpoint]: run_id: The training run identifier. Returns: - List of :class:`~twinkle_client.types.training.Checkpoint` objects. + List of :class:`~twinkle.protocol.types.training.Checkpoint` objects. Raises: - TwinkleClientError: If run not found or access denied. + TwinkleHTTPError: If run not found or access denied. """ response = self._transport.get(self._get_url(f'/training_runs/{run_id}/checkpoints')) data = response.json() @@ -348,11 +354,11 @@ def get_checkpoint_path(self, run_id: str, checkpoint_id: str) -> ParsedCheckpoi checkpoint_id: The checkpoint identifier (e.g. "weights/20240101_120000"). Returns: - :class:`~twinkle_client.types.training.ParsedCheckpointTwinklePath` with + :class:`~twinkle.protocol.types.training.ParsedCheckpointTwinklePath` with ``path`` (filesystem) and ``twinkle_path`` fields. Raises: - TwinkleClientError: If checkpoint not found or access denied. + TwinkleHTTPError: If checkpoint not found or access denied. """ response = self._transport.get(self._get_url(f'/checkpoint_path/{run_id}/{checkpoint_id}')) data = response.json() @@ -376,7 +382,7 @@ def get_checkpoint_twinkle_path(self, run_id: str, checkpoint_id: str) -> str: Twinkle path string (e.g. "twinkle://run_id/weights/checkpoint_name"). Raises: - TwinkleClientError: If checkpoint not found or access denied. + TwinkleHTTPError: If checkpoint not found or access denied. """ return self.get_checkpoint_path(run_id, checkpoint_id).twinkle_path @@ -389,10 +395,10 @@ def delete_checkpoint(self, run_id: str, checkpoint_id: str) -> DeleteCheckpoint checkpoint_id: The checkpoint identifier. Returns: - :class:`~twinkle_client.types.server.DeleteCheckpointResponse` indicating success. + :class:`~twinkle.protocol.types.server.DeleteCheckpointResponse` indicating success. Raises: - TwinkleClientError: If checkpoint not found or access denied. + TwinkleHTTPError: If checkpoint not found or access denied. """ url = self._get_url(f'/training_runs/{run_id}/checkpoints/{checkpoint_id}') response = self._transport.delete(url) @@ -411,11 +417,11 @@ def get_weights_info(self, twinkle_path: str) -> WeightsInfoResponse: twinkle_path: The twinkle:// path to the weights. Returns: - :class:`~twinkle_client.types.training.WeightsInfoResponse` with fields: + :class:`~twinkle.protocol.types.training.WeightsInfoResponse` with fields: ``training_run_id``, ``base_model``, ``model_owner``, ``is_lora``, ``lora_rank``. Raises: - TwinkleClientError: If weights not found or access denied. + TwinkleHTTPError: If weights not found or access denied. """ response = self._transport.post(self._get_url('/weights_info'), json_data={'twinkle_path': twinkle_path}) data = response.json() @@ -438,7 +444,7 @@ def get_latest_checkpoint_path(self, run_id: str) -> str | None: Filesystem path string to the latest checkpoint, or ``None`` if none exist. Raises: - TwinkleClientError: If run not found or access denied. + TwinkleHTTPError: If run not found or access denied. """ checkpoints = self.list_checkpoints(run_id) if not checkpoints: @@ -454,7 +460,7 @@ def find_training_run_by_model(self, base_model: str) -> list[TrainingRun]: base_model: The base model name to search for. Returns: - List of :class:`~twinkle_client.types.training.TrainingRun` objects + List of :class:`~twinkle.protocol.types.training.TrainingRun` objects matching the base model. """ all_runs = self.list_training_runs(limit=100) diff --git a/src/twinkle_client/model/multi_lora_transformers.py b/src/twinkle_client/model/multi_lora_transformers.py index 525a68e9..e5617737 100644 --- a/src/twinkle_client/model/multi_lora_transformers.py +++ b/src/twinkle_client/model/multi_lora_transformers.py @@ -4,11 +4,11 @@ from pathlib import Path from typing import Any, Dict, Optional +from twinkle.protocol.types import model as model_types +from twinkle.protocol.types.component import DataRef from twinkle_client._request_builder import build_request from twinkle_client.http import ClientTransport from twinkle_client.http.context import capture_transport -from twinkle_client.types import model as model_types -from twinkle_client.types.component import DataRef logger = logging.getLogger('twinkle_client') diff --git a/src/twinkle_client/processor/__init__.py b/src/twinkle_client/processor/__init__.py deleted file mode 100644 index c13f457e..00000000 --- a/src/twinkle_client/processor/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .base import InputProcessor - -__all__ = ['InputProcessor'] diff --git a/src/twinkle_client/processor/base.py b/src/twinkle_client/processor/base.py deleted file mode 100644 index d9b7430d..00000000 --- a/src/twinkle_client/processor/base.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) ModelScope Contributors. All rights reserved. -from typing import List, Literal, Optional, Union - -from twinkle import DeviceMesh -from twinkle.data_format import InputFeature -from twinkle_client.common.component_rpc import call_remote_component, create_remote_component -from twinkle_client.http import ClientTransport -from twinkle_client.http.context import capture_transport - - -class InputProcessor: - """Client wrapper for InputProcessor that calls server HTTP endpoints.""" - - def __init__( - self, - device_mesh: Optional[DeviceMesh] = None, - padding_free: bool = False, - framework: Literal['transformers', 'megatron'] = 'transformers', - *, - transport: ClientTransport | None = None, - **kwargs, - ): - self._transport = capture_transport(transport) - self.processor_id = create_remote_component( - 'processor', - 'InputProcessor', - device_mesh=device_mesh, - padding_free=padding_free, - framework=framework, - transport=self._transport, - **kwargs, - ) - - def __call__(self, inputs: Union[InputFeature, List[InputFeature]], **kwargs): - return call_remote_component(self.processor_id, '__call__', transport=self._transport, inputs=inputs, **kwargs) diff --git a/src/twinkle_client/sampler/vllm_sampler.py b/src/twinkle_client/sampler/vllm_sampler.py index 67d6595b..caa5cc3f 100644 --- a/src/twinkle_client/sampler/vllm_sampler.py +++ b/src/twinkle_client/sampler/vllm_sampler.py @@ -4,14 +4,14 @@ from typing import Any, Dict, List, Optional, Union from twinkle.data_format import InputFeature, SamplingParams, Trajectory +from twinkle.protocol.json_utils import json_safe +from twinkle.protocol.types.component import DataPlaneSampleRequest, DataRef, UnloadAdapterPathsRequest +from twinkle.protocol.types.sampler import (SamplerAddAdapterRequest, SamplerAddAdapterResponse, SampleRequest, + SampleResponseModel, SampleResponseModelList, SamplerSetTemplateRequest, + SamplerSetTemplateResponse) from twinkle_client._request_builder import build_request -from twinkle_client.common.json_utils import json_safe from twinkle_client.http import ClientTransport from twinkle_client.http.context import capture_transport -from twinkle_client.types.component import DataPlaneSampleRequest, DataRef, UnloadAdapterPathsRequest -from twinkle_client.types.sampler import (SamplerAddAdapterRequest, SamplerAddAdapterResponse, SampleRequest, - SampleResponseModel, SampleResponseModelList, SamplerSetTemplateRequest, - SamplerSetTemplateResponse) # Intentionally does NOT subclass ``twinkle.sampler.base.Sampler``: importing @@ -190,6 +190,6 @@ def set_template(self, template_cls: str, adapter_name: str = '', **kwargs) -> S def apply_patch(self, patch_cls: str, **kwargs) -> None: """Apply a patch to the model.""" - from twinkle_client.types.model import ApplyPatchRequest + from twinkle.protocol.types.model import ApplyPatchRequest body = build_request(ApplyPatchRequest, patch_cls=patch_cls, adapter_name=self.adapter_name or '', **kwargs) self._transport.post_model(f'{self.server_url}/apply_patch', body) diff --git a/src/twinkle_client/skills/bundled/twinkle-training.md b/src/twinkle_client/skills/bundled/twinkle-training.md index da582d3a..4010322a 100644 --- a/src/twinkle_client/skills/bundled/twinkle-training.md +++ b/src/twinkle_client/skills/bundled/twinkle-training.md @@ -76,7 +76,7 @@ Large models: 2× TP for 32B, 4× TP for 72B ### 1. Initialization ```python -from twinkle import init_twinkle_client +from twinkle_client import init_twinkle_client # Server Mode (primary — self-hosted) client = init_twinkle_client(base_url='http://localhost:8000', api_key='EMPTY_API_KEY') @@ -484,7 +484,7 @@ InputFeature( ```python import os from peft import LoraConfig -from twinkle import init_twinkle_client +from twinkle_client import init_twinkle_client from twinkle.dataset import Dataset, DatasetMeta from twinkle.dataloader import DataLoader from twinkle.preprocessor import SelfCognitionProcessor @@ -559,7 +559,7 @@ rt.finish(status='completed') import gc from typing import List, Dict, Any from peft import LoraConfig -from twinkle import init_twinkle_client +from twinkle_client import init_twinkle_client from twinkle.dataset import Dataset, DatasetMeta from twinkle.dataloader import DataLoader from twinkle.preprocessor import GSM8KProcessor @@ -671,7 +671,7 @@ import numpy as np import torch from typing import Any, Dict, List from peft import LoraConfig -from twinkle import init_twinkle_client +from twinkle_client import init_twinkle_client from twinkle.dataset import Dataset, DatasetMeta from twinkle.dataloader import DataLoader from twinkle.preprocessor import EmojiDPOProcessor @@ -753,7 +753,7 @@ rt.finish(status='completed') import numpy as np import torch from peft import LoraConfig -from twinkle import init_twinkle_client +from twinkle_client import init_twinkle_client from twinkle.dataset import LazyDataset, DatasetMeta from twinkle.dataloader import DataLoader from twinkle.preprocessor import Preprocessor @@ -919,7 +919,7 @@ The TUI agent's `start_server` tool handles this automatically — generates con For GRPO with Tinker API: ```python -from twinkle import init_tinker_client +from twinkle_client import init_tinker_client init_tinker_client() from tinker import ServiceClient, types diff --git a/src/twinkle_client/utils/patch_tinker.py b/src/twinkle_client/utils/patch_tinker.py index 5b1bb39c..7b408815 100644 --- a/src/twinkle_client/utils/patch_tinker.py +++ b/src/twinkle_client/utils/patch_tinker.py @@ -12,8 +12,8 @@ import os from typing import TYPE_CHECKING, Any, Dict, Mapping, Optional, Union +from twinkle.protocol.headers import build_routing_headers from twinkle_client.http.context import get_api_key, get_request_id -from twinkle_client.http.headers import build_routing_headers _patched = False _loss_fn_config_patched = False diff --git a/tests/server/contract/test_client_api_contract.py b/tests/server/contract/test_client_api_contract.py index 065017cf..3e835ad4 100644 --- a/tests/server/contract/test_client_api_contract.py +++ b/tests/server/contract/test_client_api_contract.py @@ -62,8 +62,8 @@ def test_schedule_task_and_wait_removed(): def test_new_client_types_importable(): # The only permitted client-side additions. - import twinkle_client.types.base as base - import twinkle_client.types.errors as errors + import twinkle.protocol.types.base as base + import twinkle.protocol.types.errors as errors for symbol in ('StrictRequest', 'ResponseModel', 'DataModel', 'backend_only'): assert hasattr(base, symbol) diff --git a/tests/server/contract/test_protocol_migration.py b/tests/server/contract/test_protocol_migration.py new file mode 100644 index 00000000..44bab411 --- /dev/null +++ b/tests/server/contract/test_protocol_migration.py @@ -0,0 +1,35 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Guards for the wire-contract migration to :mod:`twinkle.protocol`.""" +from __future__ import annotations + +_MIGRATED_NAMES = frozenset( + """ +AdapterRequest AddAdapterRequest AddMetricRequest AddMetricResponse ApplyPatchRequest ApplyPatchResponse +BACKEND_ONLY_KEY BackwardResponse CORE_INPUT_KEYS CalculateLossResponse CalculateMetricRequest +CalculateMetricResponse CancelRequest CancelResponse CapacityInfoResponse Checkpoint CheckpointPathResponse +CheckpointsListResponse ClientFeatures ClipGradAndStepRequest ClipGradAndStepResponse ClipGradNormRequest +ClipGradNormResponse CreateModelRequest CreateRequest CreateResponse CreateSessionRequest CreateSessionResponse Cursor +DataAppendRequest DataGetRequest DataModel DataPlaneForwardOnlyRequest DataPlaneForwardRequest DataPlaneSampleRequest +DataPutRequest DataRef DataReleaseRequest DataRowsResponse DeleteCheckpointResponse FieldRole ForwardBackwardResponse +ForwardBackwardTaskRequest ForwardOnlyRequest ForwardRequest ForwardResponse GetServerCapabilitiesResponse +GetTrainConfigsResponse HealthResponse LoadRequest LoadResponse LoraConfig LrStepRequest LrStepResponse ModelResult +OkResponse ParsedCheckpointTwinklePath ProcessorCallRequest ProcessorCallResponse ProcessorCreateRequest +ProcessorCreateResponse ProcessorHeartbeatRequest ProcessorHeartbeatResponse ProtocolLimits ResolvedLoadPath +ResponseModel ResumeFromCheckpointRequest RetrieveFutureRequest SampleRequest SampleResponseModel +SampleResponseModelList SampledSequenceModel SamplerAddAdapterRequest SamplerAddAdapterResponse SamplerCreateResponse +SamplerSetTemplateRequest SamplerSetTemplateResponse SaveRequest SaveResponse SessionHeartbeatRequest +SessionHeartbeatResponse SetLossRequest SetLossResponse SetLrSchedulerRequest SetLrSchedulerResponse SetOptimizerRequest +SetOptimizerResponse SetProcessorRequest SetProcessorResponse SetTemplateRequest SetTemplateResponse StepRequest +StepResponse StrictRequest SupportedModel TERMINAL_STATUSES TaskEnvelope TaskStatus TrainingProgressResponse TrainingRun +TrainingRunsResponse UnloadAdapterPathsRequest UploadToHubRequest VLM_TENSOR_FIELDS WeightsInfoRequest +WeightsInfoResponse WireInputBatch WireInputFeature WireInputs WireMessage WireTrajectory ZeroGradResponse backend_kwarg +backend_only declared_wire_keys export_batch fields_with_role passthrough read_backend_only read_field_role +""".split()) + + +def test_protocol_exports_match_pre_migration_snapshot() -> None: + import twinkle.protocol.types as types + + assert set(types.__all__) == _MIGRATED_NAMES + assert len(types.__all__) == 120 + assert 'ErrorResponse' not in types.__all__ diff --git a/tests/server/data_plane/test_proxy.py b/tests/server/data_plane/test_proxy.py index 550998db..2956ea32 100644 --- a/tests/server/data_plane/test_proxy.py +++ b/tests/server/data_plane/test_proxy.py @@ -3,8 +3,8 @@ import pytest from twinkle.server.data_plane.proxy import DataPlaneProxy -from twinkle_client.http.headers import H_AUTH, H_AUTH_TWINKLE, H_REQUEST_ID -from twinkle_client.types import DataRef +from twinkle.protocol.headers import H_AUTH, H_AUTH_TWINKLE, H_REQUEST_ID +from twinkle.protocol.types import DataRef class _Response: diff --git a/tests/server/data_plane/test_store.py b/tests/server/data_plane/test_store.py index 1bb001e6..8ebd315d 100644 --- a/tests/server/data_plane/test_store.py +++ b/tests/server/data_plane/test_store.py @@ -103,7 +103,7 @@ async def kv_list(partition_id): @pytest.mark.asyncio async def test_append_rejects_row_count_mismatch() -> None: - from twinkle_client.types import DataRef + from twinkle.protocol.types import DataRef store = TQDataRefStore.__new__(TQDataRefStore) ref = DataRef(ref_id='r', size=2, fields=['x']) @@ -112,7 +112,7 @@ async def test_append_rejects_row_count_mismatch() -> None: def test_partition_is_stable_and_scoped_by_data_ref() -> None: - from twinkle_client.types import DataRef + from twinkle.protocol.types import DataRef first = DataRef(ref_id='a', size=1, fields=['x']) same = DataRef(ref_id='a', size=99, fields=['other']) diff --git a/tests/server/gateway/test_openai_handlers.py b/tests/server/gateway/test_openai_handlers.py index 98f8bf19..048a4ad6 100644 --- a/tests/server/gateway/test_openai_handlers.py +++ b/tests/server/gateway/test_openai_handlers.py @@ -20,7 +20,7 @@ @pytest.fixture def mock_gateway(): """Build a minimal FastAPI app with OpenAI routes and a mock GatewayServer.""" - import twinkle_client.types as types + import twinkle.protocol.types as types from twinkle.server.gateway.openai_handlers import _register_openai_routes mock_state = AsyncMock() diff --git a/tests/server/gateway/test_proxy.py b/tests/server/gateway/test_proxy.py index e5000cf1..9b03e4a9 100644 --- a/tests/server/gateway/test_proxy.py +++ b/tests/server/gateway/test_proxy.py @@ -14,7 +14,7 @@ from unittest.mock import AsyncMock from twinkle.server.gateway.proxy import ServiceProxy -from twinkle_client.http.headers import H_MULTIPLEX, H_MULTIPLEX_LEGACY, H_REQUEST_ID +from twinkle.protocol.headers import H_MULTIPLEX, H_MULTIPLEX_LEGACY, H_REQUEST_ID def _make_request(headers: list[tuple[bytes, bytes]] | None = None) -> Request: diff --git a/tests/server/integration/e2e_helpers.py b/tests/server/integration/e2e_helpers.py index 2eca95f0..ee56bf0d 100644 --- a/tests/server/integration/e2e_helpers.py +++ b/tests/server/integration/e2e_helpers.py @@ -131,7 +131,7 @@ def create_grpo_dataset(data_slice=range(50)): def init_twinkle_client_session(): """Initialize the Twinkle client session.""" - from twinkle import init_twinkle_client + from twinkle_client import init_twinkle_client return init_twinkle_client(base_url=BASE_URL, api_key=API_KEY) @@ -205,7 +205,7 @@ def create_twinkle_sampler(): def init_tinker_client_session(): """Initialize the Tinker client session and return ServiceClient.""" - from twinkle import init_tinker_client + from twinkle_client import init_tinker_client init_tinker_client() from tinker import ServiceClient return ServiceClient(base_url=BASE_URL, api_key=API_KEY) diff --git a/tests/server/integration/test_full_cycle_e2e.py b/tests/server/integration/test_full_cycle_e2e.py index 383633e9..2e5da2ff 100644 --- a/tests/server/integration/test_full_cycle_e2e.py +++ b/tests/server/integration/test_full_cycle_e2e.py @@ -44,7 +44,8 @@ reason='Set TWINKLE_TEST_GPU_E2E=1 to run real GPU E2E tests (requires running server)', ) -from twinkle import get_logger, init_twinkle_client # noqa: E402 +from twinkle import get_logger # noqa: E402 +from twinkle_client import init_twinkle_client # noqa: E402 from twinkle.dataloader import DataLoader # noqa: E402 from twinkle.dataset import Dataset, DatasetMeta # noqa: E402 from twinkle_client.model import MultiLoraTransformersModel # noqa: E402 diff --git a/tests/server/integration/test_full_param_e2e.py b/tests/server/integration/test_full_param_e2e.py index 506a050b..8f98bc61 100644 --- a/tests/server/integration/test_full_param_e2e.py +++ b/tests/server/integration/test_full_param_e2e.py @@ -41,7 +41,8 @@ reason='Set TWINKLE_TEST_GPU_E2E=1 to run real GPU E2E tests (requires running server)', ) -from twinkle import get_logger, init_tinker_client # noqa: E402 +from twinkle import get_logger # noqa: E402 +from twinkle_client import init_tinker_client # noqa: E402 from twinkle.dataloader import DataLoader # noqa: E402 from twinkle.dataset import Dataset, DatasetMeta # noqa: E402 from twinkle.preprocessor import SelfCognitionProcessor # noqa: E402 diff --git a/tests/server/lifecycle/test_envelope_coverage.py b/tests/server/lifecycle/test_envelope_coverage.py index bf070af8..039327cb 100644 --- a/tests/server/lifecycle/test_envelope_coverage.py +++ b/tests/server/lifecycle/test_envelope_coverage.py @@ -11,7 +11,7 @@ from fastapi.routing import APIRoute from tests.server.contract.client_api_harness import build_model_app, build_sampler_app -from twinkle_client.types.lifecycle import TaskEnvelope +from twinkle.protocol.types.lifecycle import TaskEnvelope # The single exemption declaration, keyed BY APP. A flat path set would be wrong: # ``/twinkle/set_template`` and ``/twinkle/apply_patch`` exist on both apps, but only the diff --git a/tests/server/lifecycle/test_run_submit_dedup.py b/tests/server/lifecycle/test_run_submit_dedup.py index 0b16f6c4..f9e91d76 100644 --- a/tests/server/lifecycle/test_run_submit_dedup.py +++ b/tests/server/lifecycle/test_run_submit_dedup.py @@ -13,7 +13,7 @@ import pytest from twinkle.server.lifecycle.submit import run_submit -from twinkle_client.types.model import ForwardBackwardTaskRequest +from twinkle.protocol.types.model import ForwardBackwardTaskRequest class _FakeState: diff --git a/tests/server/lifecycle/test_static_guards.py b/tests/server/lifecycle/test_static_guards.py index 75c1af54..730c4ef7 100644 --- a/tests/server/lifecycle/test_static_guards.py +++ b/tests/server/lifecycle/test_static_guards.py @@ -74,7 +74,7 @@ def test_task_envelope_has_exactly_one_construction_site(): def test_server_task_status_enum_matches_client_literal(): """The two independent declarations of the task status set must not drift. - ``twinkle_client.types.lifecycle.TaskStatus`` (a Literal on the wire model) and the + ``twinkle.protocol.types.lifecycle.TaskStatus`` (a Literal on the wire model) and the server's ``TaskStatus`` enum are declared separately. ``envelope_from_record`` copies ``record['status']`` straight into ``TaskEnvelope.status``, so a value the server can write but the Literal does not list would fail pydantic validation *while serialising @@ -85,8 +85,8 @@ def test_server_task_status_enum_matches_client_literal(): from typing import get_args from twinkle.server.task_queue.types import TaskStatus as ServerTaskStatus - from twinkle_client.types.lifecycle import TERMINAL_STATUSES - from twinkle_client.types.lifecycle import TaskStatus as WireTaskStatus + from twinkle.protocol.types.lifecycle import TERMINAL_STATUSES + from twinkle.protocol.types.lifecycle import TaskStatus as WireTaskStatus server_values = {member.value for member in ServerTaskStatus} wire_values = set(get_args(WireTaskStatus)) @@ -98,7 +98,7 @@ def test_server_task_status_enum_matches_client_literal(): def test_client_future_layer_is_not_imported_by_the_server(): """``_future.py`` carries an underscore because the dependency runs one way only. - The server reverse-imports ``twinkle_client.types`` (the shared wire contract), but the + The server reverse-imports ``twinkle.protocol.types`` (the shared wire contract), but the client's polling layer is private to the client. An import in the other direction would make the server depend on client retry policy, which its own long-poll already owns. Another claim that lived only in a docstring. diff --git a/tests/server/lifecycle/test_submit_peek_e2e.py b/tests/server/lifecycle/test_submit_peek_e2e.py index cf05057e..c87aba59 100644 --- a/tests/server/lifecycle/test_submit_peek_e2e.py +++ b/tests/server/lifecycle/test_submit_peek_e2e.py @@ -17,7 +17,7 @@ from twinkle.server.task_queue.mixin import TaskQueueMixin # noqa: E402 from twinkle_client import _future # noqa: E402 from twinkle_client.exceptions import TaskFailedError # noqa: E402 -from twinkle_client.types.lifecycle import TaskEnvelope # noqa: E402 +from twinkle.protocol.types.lifecycle import TaskEnvelope # noqa: E402 class _Harness(TaskQueueMixin): diff --git a/tests/server/model/test_twinkle_async_inputs.py b/tests/server/model/test_twinkle_async_inputs.py index 3e98bf2b..ed89c328 100644 --- a/tests/server/model/test_twinkle_async_inputs.py +++ b/tests/server/model/test_twinkle_async_inputs.py @@ -4,7 +4,7 @@ from fastapi import FastAPI from starlette.requests import Request -import twinkle_client.types as types +import twinkle.protocol.types as types from twinkle.server.model.data_plane_inputs import model_result_rows from twinkle.server.model.twinkle_handlers import _register_model_twinkle_routes @@ -75,7 +75,7 @@ async def get(self, ref, *, fields=None): async def submit_and_peek(self, coro_factory, *, model_id=None, token=None, task_type=None, **schedule_kwargs): self.scheduled.append(schedule_kwargs) result = await coro_factory() - from twinkle_client.types.lifecycle import TaskEnvelope + from twinkle.protocol.types.lifecycle import TaskEnvelope return TaskEnvelope(request_id='req-test', status='completed', result=result) async def call_backend(self, fn, /, *args, admit=True, **kwargs): diff --git a/tests/server/sampler/test_twinkle_async_rows.py b/tests/server/sampler/test_twinkle_async_rows.py index 6c63ad63..4df48813 100644 --- a/tests/server/sampler/test_twinkle_async_rows.py +++ b/tests/server/sampler/test_twinkle_async_rows.py @@ -6,7 +6,7 @@ from fastapi import FastAPI from starlette.requests import Request -import twinkle_client.types as types +import twinkle.protocol.types as types from twinkle.data_format import SampledSequence, SampleResponse from twinkle.server.sampler.twinkle_handlers import ( _register_twinkle_sampler_routes, @@ -77,7 +77,7 @@ async def _on_request_start(self, _request): async def submit_background_and_peek(self, coro_factory, *, model_id=None, task_type=None): self.scheduled.append({'model_id': model_id, 'task_type': task_type}) result = await coro_factory() - from twinkle_client.types.lifecycle import TaskEnvelope + from twinkle.protocol.types.lifecycle import TaskEnvelope return TaskEnvelope(request_id='req-test', status='completed', result=result) async def call_backend(self, fn, /, *args, admit=True, **kwargs): diff --git a/tests/server/state/fake_backend.py b/tests/server/state/fake_backend.py index f59a75ae..eaac9883 100644 --- a/tests/server/state/fake_backend.py +++ b/tests/server/state/fake_backend.py @@ -41,6 +41,9 @@ async def get(self, key: str) -> Any | None: return None return self._store[key][0] + async def mget(self, keys: list[str]) -> list[Any | None]: + return [await self.get(key) for key in keys] + async def delete(self, key: str) -> None: self._store.pop(key, None) diff --git a/tests/server/state/test_error_payload.py b/tests/server/state/test_error_payload.py index 3b8ff7c1..7b251328 100644 --- a/tests/server/state/test_error_payload.py +++ b/tests/server/state/test_error_payload.py @@ -6,7 +6,7 @@ from pydantic import ValidationError from twinkle.server.task_errors import task_error_payload -from twinkle_client.types.errors import ErrorCategory, ErrorPayload +from twinkle.protocol.types.errors import ErrorCategory, ErrorPayload def test_overlong_traceback_is_trimmed_tail_kept_with_marker(): diff --git a/tests/server/state/test_future_lifecycle.py b/tests/server/state/test_future_lifecycle.py index 38994bb1..9728780f 100644 --- a/tests/server/state/test_future_lifecycle.py +++ b/tests/server/state/test_future_lifecycle.py @@ -43,6 +43,9 @@ async def set(self, key: str, value: Any, ttl: int | None = None) -> None: async def get(self, key: str) -> Any | None: return None if self._is_expired(key) else self._store[key][0] + async def mget(self, keys: list[str]) -> list[Any | None]: + return [await self.get(key) for key in keys] + async def delete(self, key: str) -> None: self._store.pop(key, None) diff --git a/tests/server/state/test_managers.py b/tests/server/state/test_managers.py index b607c9fc..242dc350 100644 --- a/tests/server/state/test_managers.py +++ b/tests/server/state/test_managers.py @@ -204,9 +204,10 @@ async def test_indexes_derived_from_backend(self, manager): avail = await manager.get_available_replica_ids(['r1', 'r2']) assert avail == ['r1', 'r2'] - # Per-token count enforces the limit using the persisted records. - count = await manager._count_models_for_token('tok1') - assert count == 2 + # The public add path enforces the per-token quota from persisted counts. + await manager.add('m3', ModelRecord(token='tok1')) + with pytest.raises(ResourceQuotaExceededError, match='Model quota exceeded'): + await manager.add('m4', ModelRecord(token='tok1')) @pytest.mark.asyncio async def test_cascade_cleanup_by_session(self, manager): diff --git a/tests/server/state/test_redis_integration.py b/tests/server/state/test_redis_integration.py index e730703a..ed6d5fd4 100644 --- a/tests/server/state/test_redis_integration.py +++ b/tests/server/state/test_redis_integration.py @@ -132,55 +132,16 @@ async def test_model_write_visible(make_state) -> None: @pytest.mark.asyncio -async def test_session_and_config(make_state) -> None: +async def test_session_shared_across_states(make_state) -> None: a = make_state() b = make_state() sid = await a.create_session({'session_id': f'sess-{uuid.uuid4().hex[:6]}'}) assert await b.get_session_last_heartbeat(sid) is not None - await a.add_config('feature_flag', {'value': 42}) - assert await b.get_config('feature_flag') == {'value': 42} - # ---------- Concurrent-write consistency --------------------------------- # -@pytest.mark.asyncio -async def test_concurrent_config_writes_no_torn_records(make_state) -> None: - """Many concurrent writes of distinct keys complete and every record - equals one of the writes (no torn / partial value).""" - a = make_state() - b = make_state() - n = 40 - payload = {f'k-{i}': {'idx': i, 'note': 'x' * 32} for i in range(n)} - - async def writer(state: ServerState, items: dict) -> None: - await asyncio.gather(*(state.add_config(k, v) for k, v in items.items())) - - half = list(payload.items())[:n // 2] - other = list(payload.items())[n // 2:] - await asyncio.gather(writer(a, dict(half)), writer(b, dict(other))) - - # Every key must read back equal to its expected payload from either side. - for k, v in payload.items(): - assert await a.get_config(k) == v, k - assert await b.get_config(k) == v, k - - -@pytest.mark.asyncio -async def test_concurrent_same_key_lands_one_of_committed(make_state) -> None: - """Two writers race on the same key — final value equals one of the - writes; no torn record.""" - a = make_state() - b = make_state() - write_a = {'who': 'a', 'payload': list(range(8))} - write_b = {'who': 'b', 'payload': list(range(8, 16))} - - await asyncio.gather(a.add_config('contended', write_a), b.add_config('contended', write_b)) - final = await a.get_config('contended') - assert final in (write_a, write_b) - - @pytest.mark.asyncio async def test_concurrent_replica_registration(make_state) -> None: a = make_state() @@ -198,9 +159,9 @@ async def test_concurrent_replica_registration(make_state) -> None: # ---------- Manager-level atomic-update guarantees ----------------------- # # # These tests pin the contract that the manager-level RMW paths -# (``SessionManager.touch``, ``ConfigManager.add_or_get``, -# ``FutureManager.store_status``) now go through ``StateBackend.update_atomic`` -# or ``set_nx``, so a concurrent retry cannot lose a freshly committed write. +# (``SessionManager.touch`` and ``FutureManager.store_status``) go through +# ``StateBackend.update_atomic``, so a concurrent retry cannot lose a freshly +# committed write. @pytest.mark.asyncio @@ -241,29 +202,6 @@ async def hammer(state: ServerState) -> None: assert final >= start, 'final heartbeat predates the test start — every write was lost' -@pytest.mark.asyncio -async def test_concurrent_add_or_get_consistent_value(make_state) -> None: - """``ConfigManager.add_or_get`` is implemented on top of ``set_nx``, - which is atomic in Redis. Two writers racing distinct values for the - same key must return the *same* committed value.""" - a = make_state() - b = make_state() - key = f'cfg-{uuid.uuid4().hex[:6]}' - write_a = {'who': 'a'} - write_b = {'who': 'b'} - - got_a, got_b = await asyncio.gather( - a.add_or_get_config(key, write_a), - b.add_or_get_config(key, write_b), - ) - # Both calls must observe the same committed value — that's the whole - # point of the SETNX-backed contract. - assert got_a == got_b - final = await a.get_config(key) - assert final == got_a - assert final in (write_a, write_b) - - @pytest.mark.asyncio async def test_concurrent_future_update_no_state_regression(make_state) -> None: """Once a future is recorded as ``completed`` a concurrent ``pending`` diff --git a/tests/server/static/test_client_architecture_imports.py b/tests/server/static/test_client_architecture_imports.py index 653666ac..6b0e0242 100644 --- a/tests/server/static/test_client_architecture_imports.py +++ b/tests/server/static/test_client_architecture_imports.py @@ -7,10 +7,10 @@ _ROOT = Path(__file__).parents[3] / 'src' _ALLOWED_SERVER_IMPORTS = ( - 'twinkle_client.types', - 'twinkle_client.http.headers', - 'twinkle_client.common.json_utils', - 'twinkle_client.common.serialize', + 'twinkle.protocol.types', + 'twinkle.protocol.headers', + 'twinkle.protocol.json_utils', + 'twinkle.protocol.serialize', ) diff --git a/tests/server/test_gateway_services.py b/tests/server/test_gateway_services.py index 8950689a..5a5d032d 100644 --- a/tests/server/test_gateway_services.py +++ b/tests/server/test_gateway_services.py @@ -37,7 +37,7 @@ def test_gateway_services_do_not_import_protocol_models(): path = Path(services.__file__) tree = ast.parse(path.read_text(), filename=str(path)) imports = {node.module for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) and node.module} - assert not any(module == 'tinker.types' or module.startswith('twinkle_client.types') for module in imports) + assert not any(module == 'tinker.types' or module.startswith('twinkle.protocol.types') for module in imports) class _FakeGateway: diff --git a/tests/server/utils/test_rate_limiter.py b/tests/server/utils/test_rate_limiter.py new file mode 100644 index 00000000..435b4361 --- /dev/null +++ b/tests/server/utils/test_rate_limiter.py @@ -0,0 +1,58 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +from __future__ import annotations + +import pytest + +from twinkle.server.task_queue.rate_limiter import RateLimiter + + +class _RecordingGauge: + + def __init__(self) -> None: + self.calls: list[tuple[int, dict[str, str]]] = [] + + def set(self, value: int, *, tags: dict[str, str]) -> None: + self.calls.append((value, tags)) + + +@pytest.mark.asyncio +async def test_zero_rps_disables_request_limit() -> None: + limiter = RateLimiter(rps_limit=0, tps_limit=100) + + assert await limiter.check_and_record('token', 1) == (True, None) + assert await limiter.check_and_record('token', 1) == (True, None) + + +@pytest.mark.asyncio +async def test_zero_tps_disables_token_limit() -> None: + limiter = RateLimiter(rps_limit=100, tps_limit=0) + + assert await limiter.check_and_record('token', 1_000_000) == (True, None) + assert await limiter.check_and_record('token', 1_000_000) == (True, None) + + +@pytest.mark.asyncio +async def test_active_token_metric_distinguishes_replicas() -> None: + gauge = _RecordingGauge() + first = RateLimiter( + rps_limit=100, + tps_limit=100, + active_tokens_gauge=gauge, + deployment_name='model', + replica_id='replica-a', + ) + second = RateLimiter( + rps_limit=100, + tps_limit=100, + active_tokens_gauge=gauge, + deployment_name='model', + replica_id='replica-b', + ) + + assert await first.check_and_record('token-a', 1) == (True, None) + assert await second.check_and_record('token-b', 1) == (True, None) + + assert gauge.calls == [ + (1, {'deployment': 'model', 'replica': 'replica-a'}), + (1, {'deployment': 'model', 'replica': 'replica-b'}), + ] diff --git a/tests/server/validation/test_preflight.py b/tests/server/validation/test_preflight.py index 330a8f4d..2912b1e8 100644 --- a/tests/server/validation/test_preflight.py +++ b/tests/server/validation/test_preflight.py @@ -16,7 +16,7 @@ from twinkle.server.lifecycle.submit import backend_kwargs, run_submit from twinkle.server.validation import BackendCapability, assert_request_supported from twinkle.server.validation.backend_compat import resolve_backend -from twinkle_client.types import model as model_types +from twinkle.protocol.types import model as model_types class _Deployment: @@ -147,7 +147,7 @@ def test_an_unset_backend_only_parameter_is_not_rejected(): def test_every_restricted_field_is_optional_with_a_none_default(): - from twinkle_client.types.base import FieldRole, fields_with_role, read_backend_only + from twinkle.protocol.types.base import FieldRole, fields_with_role, read_backend_only offenders = [] for model_cls in vars(model_types).values(): if not isinstance(model_cls, type) or not hasattr(model_cls, 'model_fields'): diff --git a/tests/server/validation/test_request_wire.py b/tests/server/validation/test_request_wire.py index c0665462..05e9bb5e 100644 --- a/tests/server/validation/test_request_wire.py +++ b/tests/server/validation/test_request_wire.py @@ -16,8 +16,8 @@ from fastapi.exceptions import RequestValidationError from twinkle.server.deployment import validation_error_handler -from twinkle_client.types import model as model_types -from twinkle_client.types.base import StrictRequest +from twinkle.protocol.types import model as model_types +from twinkle.protocol.types.base import StrictRequest @pytest.fixture(scope='module') @@ -191,7 +191,7 @@ def test_every_twinkle_route_body_is_strict(): # Regression: the sampler routes must bind the sampler-domain models, not model.py's # # ``sampler.py`` and ``model.py`` once both declared bare ``AddAdapterRequest`` / -# ``SetTemplateRequest``. Because the handler does ``import twinkle_client.types as +# ``SetTemplateRequest``. Because the handler does ``import twinkle.protocol.types as # types`` and the package ``__init__`` re-exported ``model.py`` first, # ``types.AddAdapterRequest`` resolved to *model.py*'s model -- whose ``config`` is a # ``str``, so it rejected the dict a real ``add_adapter_to_sampler`` call sends. The @@ -211,7 +211,7 @@ def _body_model(app, path: str): def test_sampler_routes_bind_sampler_domain_models(): from tests.server.contract.client_api_harness import build_sampler_app - from twinkle_client.types import sampler as sampler_types + from twinkle.protocol.types import sampler as sampler_types app = build_sampler_app() assert _body_model(app, '/twinkle/add_adapter_to_sampler') is sampler_types.SamplerAddAdapterRequest @@ -220,8 +220,8 @@ def test_sampler_routes_bind_sampler_domain_models(): def test_sampler_add_adapter_accepts_a_dict_config_where_model_rejects_it(): """The exact divergence the collision hid: the client sends ``config`` as a dict.""" - from twinkle_client.types import model as model_types - from twinkle_client.types import sampler as sampler_types + from twinkle.protocol.types import model as model_types + from twinkle.protocol.types import sampler as sampler_types # The sampler contract (``config: Any``) accepts the LoRA config dict the client sends. ok = sampler_types.SamplerAddAdapterRequest(adapter_name='a', config={'r': 8}) diff --git a/tests/server/validation/test_wire_schema.py b/tests/server/validation/test_wire_schema.py index 69b16f7b..09b18e7d 100644 --- a/tests/server/validation/test_wire_schema.py +++ b/tests/server/validation/test_wire_schema.py @@ -13,7 +13,7 @@ from twinkle.data_format.encoding import ENCODED_INPUT_KEYS, is_encoded from twinkle.processor.base import InputProcessor -from twinkle_client.types import data as wire +from twinkle.protocol.types import data as wire _INPUTS = TypeAdapter(wire.WireInputBatch) diff --git a/tests/twinkle_client/test_async_components.py b/tests/twinkle_client/test_async_components.py index 05147328..875ddc17 100644 --- a/tests/twinkle_client/test_async_components.py +++ b/tests/twinkle_client/test_async_components.py @@ -12,7 +12,7 @@ import asyncio import json -from twinkle_client.types import DataRef +from twinkle.protocol.types import DataRef class _Response: diff --git a/tests/twinkle_client/test_client_multi_turn_rollout.py b/tests/twinkle_client/test_client_multi_turn_rollout.py index bd771d80..cb144041 100644 --- a/tests/twinkle_client/test_client_multi_turn_rollout.py +++ b/tests/twinkle_client/test_client_multi_turn_rollout.py @@ -7,7 +7,7 @@ ``tests/twinkle_agentic/test_multi_turn_rollout.py`` but adapt the fake sampler to the ``twinkle_client`` HTTP contract: ``FakeClientSampler.sample()`` mirrors ``vLLMSampler.sample()`` and returns ``List[SampleResponseModel]`` (pydantic, -from ``twinkle_client.types.sampler``) whose ``sequences[0]`` carries a populated +from ``twinkle.protocol.types.sampler``) whose ``sequences[0]`` carries a populated ``new_input_feature`` so the multi-turn loop can proceed round after round. Properties covered: @@ -32,7 +32,7 @@ from twinkle_agentic.tools.base import Tool from twinkle_agentic.tools.tool_manager import ToolManager from twinkle_client.rollout.multi_turn import ClientMultiTurnRollout -from twinkle_client.types.sampler import SampledSequenceModel, SampleResponseModel +from twinkle.protocol.types.sampler import SampledSequenceModel, SampleResponseModel # ============================================================================= diff --git a/tests/twinkle_client/test_client_orchestrated_grpo.py b/tests/twinkle_client/test_client_orchestrated_grpo.py index 20953919..79628741 100644 --- a/tests/twinkle_client/test_client_orchestrated_grpo.py +++ b/tests/twinkle_client/test_client_orchestrated_grpo.py @@ -5,7 +5,7 @@ import sys from pathlib import Path -from twinkle_client.types import DataRef +from twinkle.protocol.types import DataRef MODULE_PATH = (Path(__file__).parents[2] / 'cookbook' / 'client' / 'async_rl' / 'client_orchestrated_grpo.py') diff --git a/tests/twinkle_client/test_component_rpc.py b/tests/twinkle_client/test_component_rpc.py new file mode 100644 index 00000000..f01b3bb9 --- /dev/null +++ b/tests/twinkle_client/test_component_rpc.py @@ -0,0 +1,56 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +from __future__ import annotations + +from twinkle.dataset import DatasetMeta +from twinkle.protocol.serialize import deserialize_object, serialize_object +from twinkle_client.common.component_rpc import call_remote_component, create_remote_component +from twinkle_client.http import ClientContext, ClientTransport + + +class _Response: + + def __init__(self, payload): + self._payload = payload + self.ok = True + self.status_code = 200 + self.url = 'http://server' + self.text = '' + + def json(self): + return self._payload + + +class _Session: + + def __init__(self): + self.calls = [] + + def post(self, url, **kwargs): + self.calls.append((url, kwargs)) + if url.endswith('/create'): + return _Response({'processor_id': 'pid:1'}) + return _Response({'result': 'ok'}) + + def close(self): + pass + + +def test_component_rpc_uses_transport_url_and_preserves_timeout_semantics() -> None: + session = _Session() + transport = ClientTransport(ClientContext(base_url='http://server', api_key='key'), session=session, timeout=90) + + assert create_remote_component('dataset', 'Dataset', transport=transport) == 'pid:1' + assert call_remote_component('pid:1', 'check', transport=transport) == 'ok' + assert call_remote_component('pid:1', 'check', None, transport=transport) == 'ok' + + assert session.calls[0][0] == 'http://server/api/v1/processor/twinkle/create' + assert session.calls[1][0] == 'http://server/api/v1/processor/twinkle/call' + assert session.calls[1][1]['timeout'] == 90 + assert session.calls[2][1]['timeout'] is None + + +def test_dataset_meta_data_slice_round_trips() -> None: + for data_slice in (range(1, 9, 2), [1, 3, 5]): + restored = deserialize_object(serialize_object(DatasetMeta(dataset_id='demo', data_slice=data_slice))) + assert restored.dataset_id == 'demo' + assert list(restored.data_slice) == list(data_slice) diff --git a/tests/twinkle_client/test_data_plane_async.py b/tests/twinkle_client/test_data_plane_async.py index 5a0559a1..5be9ebbd 100644 --- a/tests/twinkle_client/test_data_plane_async.py +++ b/tests/twinkle_client/test_data_plane_async.py @@ -6,7 +6,7 @@ import threading from twinkle_client.data_plane import DataPlaneClient -from twinkle_client.types import DataRef, DataRowsResponse +from twinkle.protocol.types import DataRef, DataRowsResponse def test_async_convenience_methods_delegate_to_sync_operations(monkeypatch) -> None: diff --git a/tests/twinkle_client/test_future_layer.py b/tests/twinkle_client/test_future_layer.py index f4da3c0c..b33ad137 100644 --- a/tests/twinkle_client/test_future_layer.py +++ b/tests/twinkle_client/test_future_layer.py @@ -11,8 +11,8 @@ from twinkle_client import _future from twinkle_client.exceptions import TaskFailedError, TaskRecordLostError, TaskWaitTimeoutError -from twinkle_client.types.errors import ErrorPayload -from twinkle_client.types.lifecycle import TaskEnvelope +from twinkle.protocol.types.errors import ErrorPayload +from twinkle.protocol.types.lifecycle import TaskEnvelope class _Model: @@ -101,6 +101,36 @@ def _always_503(_request_id, _transport): _future.resolve(_running(), model_cls=_Model) +def test_connection_error_is_retried_then_succeeds(monkeypatch): + monkeypatch.setattr(_future.time, 'sleep', lambda _s: None) + replies = [requests.ConnectionError('reset'), _completed({'ok': True})] + + def _next(_request_id, _transport): + value = replies.pop(0) + if isinstance(value, BaseException): + raise value + return value + + monkeypatch.setattr(_future, '_post_retrieve', _next) + out = _future.resolve(_running(), model_cls=_Model) + assert out.result == {'ok': True} + + +def test_connection_error_is_bounded_then_reraised(monkeypatch): + monkeypatch.setattr(_future.time, 'sleep', lambda _s: None) + calls = 0 + + def _always_fails(_request_id, _transport): + nonlocal calls + calls += 1 + raise requests.ConnectionError('reset') + + monkeypatch.setattr(_future, '_post_retrieve', _always_fails) + with pytest.raises(requests.ConnectionError, match='reset'): + _future.resolve(_running(), model_cls=_Model) + assert calls == _future._TRANSPORT_RETRY_MAX + 1 + + def test_non_retryable_4xx_reraises_immediately(monkeypatch): def _400(_request_id, _transport): diff --git a/tests/twinkle_client/test_import_surface.py b/tests/twinkle_client/test_import_surface.py new file mode 100644 index 00000000..fcce1339 --- /dev/null +++ b/tests/twinkle_client/test_import_surface.py @@ -0,0 +1,59 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +from __future__ import annotations + +import json +import subprocess +import sys + + +def _run_import_probe(source: str) -> dict[str, object]: + completed = subprocess.run( + [sys.executable, '-c', source], + check=True, + capture_output=True, + text=True, + ) + return json.loads(completed.stdout) + + +def test_import_twinkle_does_not_eagerly_import_client_or_torch() -> None: + result = _run_import_probe( + "import json, sys, twinkle; " + "print(json.dumps({'client': 'twinkle_client' in sys.modules, 'torch': 'torch' in sys.modules}))") + assert result == {'client': False, 'torch': False} + + +def test_twinkle_client_entry_points_remain_available_without_eager_import() -> None: + result = _run_import_probe( + "import json, sys, twinkle; " + "before = 'twinkle_client' in sys.modules; " + "from twinkle import init_tinker_client, init_twinkle_client; " + "after = 'twinkle_client' in sys.modules; " + "print(json.dumps({'before': before, 'after': after, " + "'tinker': callable(init_tinker_client), 'twinkle': callable(init_twinkle_client)}))") + assert result == {'before': False, 'after': False, 'tinker': True, 'twinkle': True} + + +def test_twinkle_client_entry_points_delegate(monkeypatch) -> None: + import twinkle + import twinkle_client + + calls = [] + monkeypatch.setattr(twinkle_client, 'init_tinker_client', lambda **kwargs: calls.append(('tinker', kwargs))) + monkeypatch.setattr(twinkle_client, 'init_twinkle_client', lambda **kwargs: ('twinkle', kwargs)) + + assert twinkle.init_tinker_client(feature=True) is None + assert calls == [('tinker', {'feature': True})] + assert twinkle.init_twinkle_client(base_url='http://server', api_key='key') == ( + 'twinkle', { + 'base_url': 'http://server', + 'api_key': 'key', + 'session_heartbeat_interval': 10, + }) + + +def test_import_twinkle_client_does_not_load_heavy_data_dependencies() -> None: + result = _run_import_probe( + "import json, sys, twinkle_client; " + "print(json.dumps({name: name in sys.modules for name in ('torch', 'datasets', 'pandas')}))") + assert result == {'torch': False, 'datasets': False, 'pandas': False} diff --git a/tests/twinkle_client/test_remote_components.py b/tests/twinkle_client/test_remote_components.py new file mode 100644 index 00000000..85657d27 --- /dev/null +++ b/tests/twinkle_client/test_remote_components.py @@ -0,0 +1,75 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +from __future__ import annotations + +import json +import subprocess +import sys + +from torch.utils.data import IterableDataset as TorchIterableDataset + +from twinkle_client.common import remote_component +from twinkle_client.dataloader import DataLoader +from twinkle_client.dataset import Dataset, IterableDataset, IterablePackingDataset, LazyDataset, PackingDataset +from twinkle_client.http import ClientContext, ClientTransport + + +class _Session: + + def close(self) -> None: + pass + + +class _CallableDataset: + + def __call__(self): + return None + + +def _transport() -> ClientTransport: + return ClientTransport(ClientContext(base_url='http://server', api_key='key'), session=_Session()) + + +def test_remote_component_binding_and_dispatch_are_shared(monkeypatch) -> None: + created: list[tuple[str, str, ClientTransport, dict]] = [] + called: list[tuple[str, str, tuple, ClientTransport, dict]] = [] + + def _create(processor_type, class_type, *, transport, **kwargs): + created.append((processor_type, class_type, transport, kwargs)) + return f'pid:{class_type}' + + def _call(processor_id, function, *args, transport, **kwargs): + called.append((processor_id, function, args, transport, kwargs)) + return 'result' + + monkeypatch.setattr(remote_component, 'create_remote_component', _create) + monkeypatch.setattr(remote_component, 'call_remote_component', _call) + transport = _transport() + + dataset = Dataset(transport=transport) + loader = DataLoader(_CallableDataset(), transport=transport) + + assert dataset.check(flag=True) == 'result' + assert loader.get_state() == 'result' + assert created[0][:3] == ('dataset', 'Dataset', transport) + assert created[1][:3] == ('dataloader', 'DataLoader', transport) + assert called[0] == ('pid:Dataset', 'check', (), transport, {'flag': True}) + assert called[1] == ('pid:DataLoader', 'get_state', (), transport, {}) + + +def test_dataset_method_surfaces_and_iterable_mro() -> None: + assert 'map' not in LazyDataset.__dict__ + assert hasattr(LazyDataset, 'map') + assert hasattr(PackingDataset, 'map') + assert '__len__' not in IterableDataset.__dict__ + assert '__getitem__' not in IterableDataset.__dict__ + assert IterableDataset.__mro__[1] is TorchIterableDataset + assert IterablePackingDataset.__mro__[1] is TorchIterableDataset + + +def test_dataloader_import_does_not_load_transformers() -> None: + source = ( + "import json, sys, twinkle_client.dataloader; " + "print(json.dumps({'transformers': 'transformers' in sys.modules}))" + ) + result = subprocess.run([sys.executable, '-c', source], check=True, capture_output=True, text=True) + assert json.loads(result.stdout) == {'transformers': False} diff --git a/tests/twinkle_client/test_request_builder.py b/tests/twinkle_client/test_request_builder.py index 6d743e66..9617b04b 100644 --- a/tests/twinkle_client/test_request_builder.py +++ b/tests/twinkle_client/test_request_builder.py @@ -12,10 +12,10 @@ from typing import Dict from twinkle_client._request_builder import build_request, request_json, to_wire_value -from twinkle_client.common.serialize import serialize_object +from twinkle.protocol.serialize import serialize_object from twinkle_client.exceptions import TwinkleClientValidationError -from twinkle_client.types import model as model_types -from twinkle_client.types.base import StrictRequest, passthrough +from twinkle.protocol.types import model as model_types +from twinkle.protocol.types.base import StrictRequest, passthrough @pytest.fixture(autouse=True) diff --git a/tests/twinkle_client/test_transport.py b/tests/twinkle_client/test_transport.py index 43b6c34c..917c869b 100644 --- a/tests/twinkle_client/test_transport.py +++ b/tests/twinkle_client/test_transport.py @@ -1,8 +1,11 @@ # Copyright (c) ModelScope Contributors. All rights reserved. from __future__ import annotations +import logging +import pytest + from twinkle_client.http import ClientContext, ClientTransport -from twinkle_client.http.context import set_default_transport +from twinkle_client.http.context import capture_transport, clear_default_transport, set_default_transport from twinkle_client.manager import TwinkleClient @@ -108,6 +111,56 @@ def test_http_public_api_has_no_legacy_context_getters_or_setters(): } & set(http.__all__)) +def test_default_transport_fallback_logs_info_once(monkeypatch, caplog): + import twinkle_client.http.context as context + + monkeypatch.setattr(context, '_default_transport', None) + with caplog.at_level(logging.INFO, logger='twinkle_client'): + transport = capture_transport() + assert capture_transport() is transport + records = [record for record in caplog.records if 'No explicit Twinkle client configured' in record.message] + assert len(records) == 1 + assert transport.context.base_url in records[0].message + transport.close() + clear_default_transport(transport) + + +def test_publish_blocks_rebind_and_replacement_does_not_close_old(caplog): + first = _transport('first') + first.bind_context(ClientContext(base_url='http://bound', api_key='key')) + set_default_transport(first) + + with pytest.raises(RuntimeError, match='published'): + first.bind_context(ClientContext(base_url='http://too-late', api_key='key')) + + second = _transport('second') + with caplog.at_level(logging.WARNING, logger='twinkle_client'): + set_default_transport(second) + assert any('Replacing default Twinkle transport' in record.message for record in caplog.records) + assert not first.closed + clear_default_transport(second) + first.close() + second.close() + + +def test_transport_adapter_configuration_and_post_retry_boundary(): + transport = ClientTransport(ClientContext(base_url='http://pool', api_key='key')) + adapter = transport._session.get_adapter('http://') + assert adapter._pool_maxsize == 32 + assert adapter._pool_block is True + assert adapter.max_retries.allowed_methods == frozenset({'GET', 'DELETE'}) + assert 'POST' not in adapter.max_retries.allowed_methods + transport.close() + + +def test_client_closes_distinct_heartbeat_transport(): + main = _transport('main') + heartbeat = _transport('heartbeat') + client = TwinkleClient(transport=main, heartbeat_transport=heartbeat) + client.close() + assert main.closed and heartbeat.closed + + class _CapabilitiesResponse(_Response): def json(self): @@ -130,7 +183,7 @@ def _capability_client(name: str) -> TwinkleClient: def test_capability_cache_is_per_transport_and_not_process_global(): - from twinkle_client.types.server import GetServerCapabilitiesResponse + from twinkle.protocol.types.server import GetServerCapabilitiesResponse client_a = _capability_client('alpha') client_b = _capability_client('beta') diff --git a/tests/twinkle_client/test_types_contract.py b/tests/twinkle_client/test_types_contract.py index d5cfc9a7..7dbaa25d 100644 --- a/tests/twinkle_client/test_types_contract.py +++ b/tests/twinkle_client/test_types_contract.py @@ -16,8 +16,9 @@ import twinkle from twinkle.server.task_queue.types import QueueState -from twinkle_client.types.errors import QueueStateLiteral -from twinkle_client.types.server import GetServerCapabilitiesResponse +from twinkle.protocol.types import model as model_types +from twinkle.protocol.types.errors import QueueStateLiteral +from twinkle.protocol.types.server import GetServerCapabilitiesResponse _TWINKLE_SRC = pathlib.Path(twinkle.__file__).resolve().parent _LEGACY_PUBLIC_NAME_OVERLAP = frozenset({ @@ -73,6 +74,31 @@ def test_capabilities_response_ignores_future_fields(): assert response.limits.max_batch_size == 8 +def test_data_plane_forward_only_has_no_seq_id() -> None: + fields = model_types.DataPlaneForwardOnlyRequest.model_fields + assert 'seq_id' not in fields + assert 'seq_id' in model_types.DataPlaneForwardRequest.model_fields + + +def test_void_response_names_are_canonical_ok_response_aliases() -> None: + for name in ( + 'BackwardResponse', + 'StepResponse', + 'ZeroGradResponse', + 'LrStepResponse', + 'SetLossResponse', + 'SetOptimizerResponse', + 'SetLrSchedulerResponse', + 'LoadResponse', + 'SetTemplateResponse', + 'SetProcessorResponse', + 'ClipGradAndStepResponse', + 'ApplyPatchResponse', + 'AddMetricResponse', + ): + assert getattr(model_types, name) is model_types.OkResponse + + def _origin(module: str | None) -> str | None: """Classify an import's source module as 'tinker', 'twinkle_client', or None.""" if not module: @@ -109,9 +135,9 @@ def _binding_collisions(tree: ast.AST) -> set[str]: def test_public_name_overlap_does_not_grow(): import tinker.types - import twinkle_client.types + import twinkle.protocol.types - overlap = {name for name in set(dir(tinker.types)) & set(dir(twinkle_client.types)) if not name.startswith('_')} + overlap = {name for name in set(dir(tinker.types)) & set(dir(twinkle.protocol.types)) if not name.startswith('_')} assert overlap == _LEGACY_PUBLIC_NAME_OVERLAP From 4f13c1f017cb5c2c8c97842a872838c2f4ff570d Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 21 Sep 2026 10:08:22 +0000 Subject: [PATCH 30/32] fix: preserve client entry compatibility and unify errors --- README.md | 4 +-- README_ZH.md | 4 +-- .../async_rl/client_orchestrated_grpo.py | 3 +- cookbook/client/tinker/dpo.py | 4 +-- cookbook/client/tinker/lora.py | 2 +- cookbook/client/tinker/multi_modal.py | 4 +-- cookbook/client/tinker/sample.py | 2 +- cookbook/client/tinker/self_cognition.py | 4 +-- cookbook/client/tinker/short_math_grpo.py | 2 +- cookbook/client/tinker/upload_to_hub.py | 2 +- cookbook/client/twinkle/dpo.py | 2 +- cookbook/client/twinkle/embedding.py | 2 +- cookbook/client/twinkle/multi_modal.py | 2 +- cookbook/client/twinkle/multi_turn_rollout.py | 2 +- cookbook/client/twinkle/sample.py | 2 +- cookbook/client/twinkle/self_cognition.py | 2 +- cookbook/client/twinkle/short_math_grpo.py | 2 +- cookbook/client/twinkle/upload_to_hub.py | 2 +- .../Usage Guide/Embedding-Training.md | 2 +- .../Usage Guide/Introduction-with-Qwen3.5.md | 6 ++-- docs/source_en/Usage Guide/Quick-Start.md | 4 +-- .../Tinker-Compatible-Client.md | 8 ++--- .../Server and Client/Twinkle-Client.md | 8 ++--- .../Usage Guide/Train-as-a-Service.md | 6 ++-- .../Embedding\350\256\255\347\273\203.md" | 2 +- ...00\344\275\263\345\256\236\350\267\265.md" | 6 ++-- ...53\351\200\237\345\274\200\345\247\213.md" | 4 +-- ...71\345\256\242\346\210\267\347\253\257.md" | 8 ++--- ...le\345\256\242\346\210\267\347\253\257.md" | 8 ++--- ...55\347\273\203\346\234\215\345\212\241.md" | 6 ++-- notebook/dpo.ipynb | 8 ++--- notebook/multi_modal.ipynb | 6 ++-- notebook/sample.ipynb | 2 +- notebook/self_cognition.ipynb | 6 ++-- notebook/short_math_grpo.ipynb | 8 +++-- src/twinkle/__init__.py | 4 +-- src/twinkle/server/deployment.py | 17 +++++----- src/twinkle/server/task_errors.py | 28 ++++++++++++++--- src/twinkle_client/__init__.py | 2 +- .../skills/bundled/twinkle-training.md | 14 ++++----- tests/server/integration/e2e_helpers.py | 4 +-- .../server/integration/test_full_cycle_e2e.py | 2 +- .../server/integration/test_full_param_e2e.py | 2 +- .../integration/test_mock_mode_startup.py | 4 +-- .../integration/test_nccl_safe_tinker_e2e.py | 2 +- .../integration/test_nccl_safe_twinkle_e2e.py | 2 +- .../lifecycle/test_preflight_rejection.py | 17 ++++++++++ tests/server/state/test_error_payload.py | 31 ++++++++++++++++++- .../test_deployment_exception_boundary.py | 30 ++++++++++++++++++ tests/server/validation/test_request_wire.py | 12 +++++++ 50 files changed, 214 insertions(+), 102 deletions(-) diff --git a/README.md b/README.md index dc73830b..6f4b05a0 100644 --- a/README.md +++ b/README.md @@ -252,11 +252,11 @@ if __name__ == '__main__': import os from tqdm import tqdm from tinker import types -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum base_model = 'ms://Qwen/Qwen3.8-27B' base_url='your-base-url' diff --git a/README_ZH.md b/README_ZH.md index 8cb110bb..3d5f05fa 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -241,11 +241,11 @@ if __name__ == '__main__': import os from tqdm import tqdm from tinker import types -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum base_model = 'ms://Qwen/Qwen3.8-27B' base_url='your-base-url' diff --git a/cookbook/client/async_rl/client_orchestrated_grpo.py b/cookbook/client/async_rl/client_orchestrated_grpo.py index 99f010be..cce5ccfd 100644 --- a/cookbook/client/async_rl/client_orchestrated_grpo.py +++ b/cookbook/client/async_rl/client_orchestrated_grpo.py @@ -15,7 +15,8 @@ from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor.llm import GSM8KProcessor from twinkle.reward import GSM8KAccuracyReward -from twinkle_client import DataPlaneClient, init_twinkle_client +from twinkle import init_twinkle_client +from twinkle_client import DataPlaneClient from twinkle_client.async_rl import Worker, WorkerPipeline from twinkle.protocol.json_utils import json_safe from twinkle_client.model import MultiLoraTransformersModel diff --git a/cookbook/client/tinker/dpo.py b/cookbook/client/tinker/dpo.py index 16e2de50..3f6e25d0 100644 --- a/cookbook/client/tinker/dpo.py +++ b/cookbook/client/tinker/dpo.py @@ -24,11 +24,11 @@ from tinker import types from twinkle import get_logger -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.dataset import Dataset, DatasetMeta, LazyDataset from twinkle.dataloader import DataLoader from twinkle.preprocessor import EmojiDPOProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum logger = get_logger() diff --git a/cookbook/client/tinker/lora.py b/cookbook/client/tinker/lora.py index da1a7abf..2b2bbe86 100644 --- a/cookbook/client/tinker/lora.py +++ b/cookbook/client/tinker/lora.py @@ -13,7 +13,7 @@ import os # Step 2: Initialize Tinker client before importing ServiceClient -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client init_tinker_client() diff --git a/cookbook/client/tinker/multi_modal.py b/cookbook/client/tinker/multi_modal.py index d26c5237..594c4bb8 100644 --- a/cookbook/client/tinker/multi_modal.py +++ b/cookbook/client/tinker/multi_modal.py @@ -17,7 +17,7 @@ # ============================================================================= # Step 1: Initialize Tinker client (MUST be done before importing ServiceClient) # ============================================================================= -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client init_tinker_client() @@ -30,7 +30,7 @@ from twinkle.preprocessor import Preprocessor from twinkle.dataset import DatasetMeta, LazyDataset from twinkle.dataloader import DataLoader -from twinkle.server.common import input_feature_to_datum # Key: converts InputFeature -> Datum +from twinkle.server.model.tinker_datum import input_feature_to_datum # Key: converts InputFeature -> Datum from twinkle import get_logger logger = get_logger() diff --git a/cookbook/client/tinker/sample.py b/cookbook/client/tinker/sample.py index 69380388..2d94c95a 100644 --- a/cookbook/client/tinker/sample.py +++ b/cookbook/client/tinker/sample.py @@ -9,7 +9,7 @@ from twinkle.data_format import Message, Trajectory from twinkle.template import Template, Qwen3_5Template -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client # Step 1: Initialize Tinker client init_tinker_client() diff --git a/cookbook/client/tinker/self_cognition.py b/cookbook/client/tinker/self_cognition.py index a785087e..3a1cd75b 100644 --- a/cookbook/client/tinker/self_cognition.py +++ b/cookbook/client/tinker/self_cognition.py @@ -10,13 +10,13 @@ import numpy as np from tqdm import tqdm from tinker import types -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.data_format import Message, Trajectory from twinkle.template import Template from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum # Initialize the Tinker client before importing ServiceClient init_tinker_client() diff --git a/cookbook/client/tinker/short_math_grpo.py b/cookbook/client/tinker/short_math_grpo.py index 168d759e..10eabf41 100644 --- a/cookbook/client/tinker/short_math_grpo.py +++ b/cookbook/client/tinker/short_math_grpo.py @@ -24,7 +24,7 @@ from tinker import types from typing import List, Tuple, Dict, Any -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle import get_logger from twinkle.advantage import GRPOAdvantage from twinkle.dataloader import DataLoader diff --git a/cookbook/client/tinker/upload_to_hub.py b/cookbook/client/tinker/upload_to_hub.py index 32b860c9..d3582214 100644 --- a/cookbook/client/tinker/upload_to_hub.py +++ b/cookbook/client/tinker/upload_to_hub.py @@ -23,7 +23,7 @@ import os from twinkle import get_logger -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle_client.model import MultiLoraTransformersModel logger = get_logger() diff --git a/cookbook/client/twinkle/dpo.py b/cookbook/client/twinkle/dpo.py index 4b76e59d..86922921 100644 --- a/cookbook/client/twinkle/dpo.py +++ b/cookbook/client/twinkle/dpo.py @@ -13,7 +13,7 @@ from typing import Any, Dict, List from twinkle import get_logger -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import EmojiDPOProcessor diff --git a/cookbook/client/twinkle/embedding.py b/cookbook/client/twinkle/embedding.py index bc1211d0..7c286a57 100644 --- a/cookbook/client/twinkle/embedding.py +++ b/cookbook/client/twinkle/embedding.py @@ -29,7 +29,7 @@ from typing import Any, Dict, List from twinkle import get_logger -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.template import Qwen3_5Template dotenv.load_dotenv('.env') diff --git a/cookbook/client/twinkle/multi_modal.py b/cookbook/client/twinkle/multi_modal.py index 20354f23..1cea9bb5 100644 --- a/cookbook/client/twinkle/multi_modal.py +++ b/cookbook/client/twinkle/multi_modal.py @@ -12,7 +12,7 @@ from peft import LoraConfig from twinkle import get_logger -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.data_format import Message, Trajectory from twinkle.dataloader import DataLoader from twinkle.dataset import DatasetMeta, LazyDataset diff --git a/cookbook/client/twinkle/multi_turn_rollout.py b/cookbook/client/twinkle/multi_turn_rollout.py index ba160286..942f572b 100644 --- a/cookbook/client/twinkle/multi_turn_rollout.py +++ b/cookbook/client/twinkle/multi_turn_rollout.py @@ -27,7 +27,7 @@ from typing import Any, Dict, List, Tuple from twinkle import get_logger -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.advantage import GRPOAdvantage from twinkle.data_format import SamplingParams from twinkle.template import Qwen3_5Template diff --git a/cookbook/client/twinkle/sample.py b/cookbook/client/twinkle/sample.py index 8449b4e5..7e999462 100644 --- a/cookbook/client/twinkle/sample.py +++ b/cookbook/client/twinkle/sample.py @@ -14,7 +14,7 @@ from transformers import AutoTokenizer from twinkle import get_logger -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client dotenv.load_dotenv('.env') logger = get_logger() diff --git a/cookbook/client/twinkle/self_cognition.py b/cookbook/client/twinkle/self_cognition.py index 14547520..76f44235 100644 --- a/cookbook/client/twinkle/self_cognition.py +++ b/cookbook/client/twinkle/self_cognition.py @@ -10,7 +10,7 @@ from peft import LoraConfig from twinkle import get_logger -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta diff --git a/cookbook/client/twinkle/short_math_grpo.py b/cookbook/client/twinkle/short_math_grpo.py index 7ef88036..21494ab0 100644 --- a/cookbook/client/twinkle/short_math_grpo.py +++ b/cookbook/client/twinkle/short_math_grpo.py @@ -28,7 +28,7 @@ from typing import Any, Dict, List, Tuple from twinkle import get_logger -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.advantage import GRPOAdvantage from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta diff --git a/cookbook/client/twinkle/upload_to_hub.py b/cookbook/client/twinkle/upload_to_hub.py index 5dc510e2..7a2bc2fd 100644 --- a/cookbook/client/twinkle/upload_to_hub.py +++ b/cookbook/client/twinkle/upload_to_hub.py @@ -20,7 +20,7 @@ import os from twinkle import get_logger -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client dotenv.load_dotenv('.env') logger = get_logger() diff --git a/docs/source_en/Usage Guide/Embedding-Training.md b/docs/source_en/Usage Guide/Embedding-Training.md index d0517e18..4198f699 100644 --- a/docs/source_en/Usage Guide/Embedding-Training.md +++ b/docs/source_en/Usage Guide/Embedding-Training.md @@ -128,7 +128,7 @@ The difference from the bare library is that the client passes **class-name stri ```python from peft import LoraConfig -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle_client.model import MultiLoraTransformersModel # --- Connect to the running Twinkle server --- diff --git a/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md b/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md index ed8825ce..e61ed919 100644 --- a/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md +++ b/docs/source_en/Usage Guide/Introduction-with-Qwen3.5.md @@ -368,7 +368,7 @@ from peft import LoraConfig from twinkle import get_logger from twinkle.dataset import DatasetMeta -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle_client.dataloader import DataLoader from twinkle_client.dataset import Dataset from twinkle_client.model import MultiLoraTransformersModel @@ -454,11 +454,11 @@ import os from tinker import types from tqdm import tqdm -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum # Initialize Tinker client (must be called before importing ServiceClient) init_tinker_client() diff --git a/docs/source_en/Usage Guide/Quick-Start.md b/docs/source_en/Usage Guide/Quick-Start.md index 69953a9a..0b722de0 100644 --- a/docs/source_en/Usage Guide/Quick-Start.md +++ b/docs/source_en/Usage Guide/Quick-Start.md @@ -514,7 +514,7 @@ from twinkle import get_logger from twinkle.advantage import GRPOAdvantage from twinkle.dataset import DatasetMeta from twinkle.metric import CompletionRewardMetric -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle_client.dataloader import DataLoader from twinkle_client.dataset import Dataset from twinkle_client.model import MultiLoraTransformersModel @@ -761,7 +761,7 @@ from tinker import ServiceClient from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum # The base model to fine-tune / evaluate base_model = 'ms://Qwen/Qwen3.5-4B' diff --git a/docs/source_en/Usage Guide/Server and Client/Tinker-Compatible-Client.md b/docs/source_en/Usage Guide/Server and Client/Tinker-Compatible-Client.md index 37860646..df9d1169 100644 --- a/docs/source_en/Usage Guide/Server and Client/Tinker-Compatible-Client.md +++ b/docs/source_en/Usage Guide/Server and Client/Tinker-Compatible-Client.md @@ -6,7 +6,7 @@ The Tinker Client is suitable for scenarios with existing Tinker training code. ```python # Initialize Tinker client before importing ServiceClient -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client init_tinker_client() @@ -41,11 +41,11 @@ import os import numpy as np from tqdm import tqdm from tinker import types -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum # Step 1: Initialize Tinker client before importing ServiceClient init_tinker_client() @@ -132,7 +132,7 @@ You can also load saved checkpoints for inference: ```python import os from tinker import types -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.data_format import Message, Trajectory from twinkle.template import Template diff --git a/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md b/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md index a41f5af2..3f3cd365 100644 --- a/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md +++ b/docs/source_en/Usage Guide/Server and Client/Twinkle-Client.md @@ -5,7 +5,7 @@ Twinkle Client is the native client, designed with the philosophy: **Change `fro ## Initialization ```python -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client # Initialize client, connect to Twinkle Server client = init_twinkle_client( @@ -50,7 +50,7 @@ from twinkle.model import MultiLoraTransformersModel # DataLoader and Dataset can be imported from either local twinkle or remote twinkle_client from twinkle.dataloader import DataLoader # or: from twinkle_client.dataloader import DataLoader from twinkle.dataset import Dataset # or: from twinkle_client.dataset import Dataset -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client client = init_twinkle_client(base_url=base_url, api_key=api_key) model = client.model(f'ms://{base_model}') @@ -67,7 +67,7 @@ dotenv.load_dotenv('.env') from peft import LoraConfig from twinkle import get_logger from twinkle.dataset import DatasetMeta -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client # DataLoader and Dataset can be imported from either local twinkle or remote twinkle_client from twinkle.dataloader import DataLoader @@ -222,7 +222,7 @@ The examples above are all single-turn training. If you want to do **multi-turn ```python from peft import LoraConfig -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.advantage import GRPOAdvantage from twinkle.data_format import SamplingParams from twinkle.template import Qwen3_5Template diff --git a/docs/source_en/Usage Guide/Train-as-a-Service.md b/docs/source_en/Usage Guide/Train-as-a-Service.md index e1ae3913..c07e072e 100644 --- a/docs/source_en/Usage Guide/Train-as-a-Service.md +++ b/docs/source_en/Usage Guide/Train-as-a-Service.md @@ -24,11 +24,11 @@ Sample code: import os from tqdm import tqdm from tinker import types -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum base_model = 'ms://Qwen/Qwen3.8-27B' base_url='https://www.modelscope.cn/twinkle' @@ -72,7 +72,7 @@ from tinker import types from twinkle.data_format import Message, Trajectory from twinkle.template import Template -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client # Step 1: Initialize Tinker client init_tinker_client() diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Embedding\350\256\255\347\273\203.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Embedding\350\256\255\347\273\203.md" index 7e3829ef..b3b5d308 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Embedding\350\256\255\347\273\203.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Embedding\350\256\255\347\273\203.md" @@ -128,7 +128,7 @@ set_processor('InputProcessor') ```python from peft import LoraConfig -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle_client.model import MultiLoraTransformersModel # --- Connect to the running Twinkle server --- diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" index 2130e742..d6fc45d2 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/Qwen3.5\346\234\200\344\275\263\345\256\236\350\267\265.md" @@ -368,7 +368,7 @@ from peft import LoraConfig from twinkle import get_logger from twinkle.dataset import DatasetMeta -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle_client.dataloader import DataLoader from twinkle_client.dataset import Dataset from twinkle_client.model import MultiLoraTransformersModel @@ -454,11 +454,11 @@ import os from tinker import types from tqdm import tqdm -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum # 初始化 Tinker 客户端（必须在导入 ServiceClient 之前） init_tinker_client() diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\345\277\253\351\200\237\345\274\200\345\247\213.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\345\277\253\351\200\237\345\274\200\345\247\213.md" index 0cfaa4bb..2efa4cc8 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\345\277\253\351\200\237\345\274\200\345\247\213.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\345\277\253\351\200\237\345\274\200\345\247\213.md" @@ -515,7 +515,7 @@ from twinkle import get_logger from twinkle.advantage import GRPOAdvantage from twinkle.dataset import DatasetMeta from twinkle.metric import CompletionRewardMetric -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle_client.dataloader import DataLoader from twinkle_client.dataset import Dataset from twinkle_client.model import MultiLoraTransformersModel @@ -762,7 +762,7 @@ from tinker import ServiceClient from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum # The base model to fine-tune / evaluate base_model = 'Qwen/Qwen3.5-4B' diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Tinker\345\205\274\345\256\271\345\256\242\346\210\267\347\253\257.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Tinker\345\205\274\345\256\271\345\256\242\346\210\267\347\253\257.md" index 885bf59a..8690ff05 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Tinker\345\205\274\345\256\271\345\256\242\346\210\267\347\253\257.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Tinker\345\205\274\345\256\271\345\256\242\346\210\267\347\253\257.md" @@ -6,7 +6,7 @@ Tinker Client 适用于已有 Tinker 训练代码的场景。通过 `init_tinker ```python # 在导入 ServiceClient 之前，先初始化 Tinker 客户端 -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client init_tinker_client() @@ -41,11 +41,11 @@ import os import numpy as np from tqdm import tqdm from tinker import types -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum # Step 1: 在导入 ServiceClient 之前，先初始化 Tinker 客户端 init_tinker_client() @@ -132,7 +132,7 @@ for i, seq in enumerate(result.sequences): ```python import os from tinker import types -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.data_format import Message, Trajectory from twinkle.template import Template diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" index ad9dd952..ed60681d 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\346\234\215\345\212\241\347\253\257\345\222\214\345\256\242\346\210\267\347\253\257/Twinkle\345\256\242\346\210\267\347\253\257.md" @@ -5,7 +5,7 @@ Twinkle Client 是原生客户端，设计理念是：**将 `from twinkle import ## 初始化 ```python -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client # 初始化客户端，连接到 Twinkle Server client = init_twinkle_client( @@ -50,7 +50,7 @@ from twinkle.model import MultiLoraTransformersModel # DataLoader 和 Dataset 使用本地 twinkle 或远端 twinkle_client 均可 from twinkle.dataloader import DataLoader # 或 from twinkle_client.dataloader import DataLoader from twinkle.dataset import Dataset # 或 from twinkle_client.dataset import Dataset -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client client = init_twinkle_client(base_url=base_url, api_key=api_key) model = client.model(f'ms://{base_model}') @@ -67,7 +67,7 @@ dotenv.load_dotenv('.env') from peft import LoraConfig from twinkle import get_logger from twinkle.dataset import DatasetMeta -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client # DataLoader 和 Dataset 使用本地 twinkle 或远端 twinkle_client 均可 from twinkle.dataloader import DataLoader @@ -221,7 +221,7 @@ model.set_lr_scheduler('default', lr_decay_steps=1000, max_lr=1e-4) ```python from peft import LoraConfig -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.advantage import GRPOAdvantage from twinkle.data_format import SamplingParams from twinkle.template import Qwen3_5Template diff --git "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" index f7a77db0..fb515944 100644 --- "a/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" +++ "b/docs/source_zh/\344\275\277\347\224\250\346\214\207\345\274\225/\350\256\255\347\273\203\346\234\215\345\212\241.md" @@ -25,11 +25,11 @@ import os from tqdm import tqdm from tinker import types -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client from twinkle.dataloader import DataLoader from twinkle.dataset import Dataset, DatasetMeta from twinkle.preprocessor import SelfCognitionProcessor -from twinkle.server.common import input_feature_to_datum +from twinkle.server.model.tinker_datum import input_feature_to_datum base_model = 'ms://Qwen/Qwen3.8-27B' base_url='https://www.modelscope.cn/twinkle' @@ -73,7 +73,7 @@ from tinker import types from twinkle.data_format import Message, Trajectory from twinkle.template import Template -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client # Step 1: Initialize Tinker client init_tinker_client() diff --git a/notebook/dpo.ipynb b/notebook/dpo.ipynb index db63ad44..9d575c09 100644 --- a/notebook/dpo.ipynb +++ b/notebook/dpo.ipynb @@ -140,11 +140,11 @@ "\n", "from tinker import types\n", "from getpass import getpass\n", - "from twinkle import get_logger\nfrom twinkle_client import init_tinker_client\n", + "from twinkle import get_logger\nfrom twinkle import init_tinker_client\n", "from twinkle.dataset import Dataset, DatasetMeta, LazyDataset\n", "from twinkle.dataloader import DataLoader\n", "from twinkle.preprocessor import EmojiDPOProcessor\n", - "from twinkle.server.common import input_feature_to_datum\n", + "from twinkle.server.model.tinker_datum import input_feature_to_datum\n", "\n", "logger = get_logger()\n", "\n", @@ -433,7 +433,7 @@ "metadata": {}, "outputs": [], "source": [ - "from twinkle_client import init_twinkle_client\n", + "from twinkle import init_twinkle_client\n", "from twinkle_client.model import MultiLoraTransformersModel\n", "# 步骤 1：初始化 Twinkle 客户端。\n", "# Tinker 检查点（twinkle:// 路径）由同一检查点服务解析\n", @@ -478,7 +478,7 @@ "source": [ "# 推理示例（使用线上服务，无需本地 GPU）\n", "from tinker import types\n", - "from twinkle import get_logger\nfrom twinkle_client import init_tinker_client\n", + "from twinkle import get_logger\nfrom twinkle import init_tinker_client\n", "from twinkle.data_format import Message, Trajectory\n", "from twinkle.template import Template\n", "\n", diff --git a/notebook/multi_modal.ipynb b/notebook/multi_modal.ipynb index 2715192a..17221cf9 100644 --- a/notebook/multi_modal.ipynb +++ b/notebook/multi_modal.ipynb @@ -124,7 +124,7 @@ "from twinkle.data_format import Trajectory, Message\n", "from twinkle.preprocessor import Preprocessor\n", "from twinkle.dataset import DatasetMeta\n", - "from twinkle_client import init_twinkle_client\n", + "from twinkle import init_twinkle_client\n", "from twinkle.dataloader import DataLoader\n", "from twinkle.dataset import LazyDataset\n", "from twinkle_client.model import MultiLoraTransformersModel\n", @@ -360,7 +360,7 @@ "metadata": {}, "outputs": [], "source": [ - "from twinkle_client import init_twinkle_client\n", + "from twinkle import init_twinkle_client\n", "from twinkle_client.model import MultiLoraTransformersModel\n", "# 步骤 1：初始化 Twinkle 客户端。\n", "# Tinker 检查点（twinkle:// 路径）由同一检查点服务解析\n", @@ -407,7 +407,7 @@ "source": [ "# 推理示例（使用线上服务，无需本地 GPU）\n", "from tinker import types\n", - "from twinkle import get_logger\nfrom twinkle_client import init_tinker_client\n", + "from twinkle import get_logger\nfrom twinkle import init_tinker_client\n", "from twinkle.data_format import Message, Trajectory\n", "from twinkle.template import Qwen3_5Template\n", "\n", diff --git a/notebook/sample.ipynb b/notebook/sample.ipynb index 7a6b641d..22d56cb0 100644 --- a/notebook/sample.ipynb +++ b/notebook/sample.ipynb @@ -102,7 +102,7 @@ "source": [ "from tinker import types\n", "from getpass import getpass\n", - "from twinkle import get_logger\nfrom twinkle_client import init_tinker_client\n", + "from twinkle import get_logger\nfrom twinkle import init_tinker_client\n", "from twinkle.data_format import Message, Trajectory\n", "from twinkle.template import Template\n", "\n", diff --git a/notebook/self_cognition.ipynb b/notebook/self_cognition.ipynb index 61c0cd2c..d6c3b109 100644 --- a/notebook/self_cognition.ipynb +++ b/notebook/self_cognition.ipynb @@ -118,13 +118,13 @@ "import os\n", "from tqdm import tqdm\n", "from tinker import types\n", - "from twinkle_client import init_tinker_client\n", + "from twinkle import init_tinker_client\n", "from twinkle.data_format import Message, Trajectory\n", "from twinkle.template import Template\n", "from twinkle.dataloader import DataLoader\n", "from twinkle.dataset import Dataset, DatasetMeta\n", "from twinkle.preprocessor import SelfCognitionProcessor\n", - "from twinkle.server.common import input_feature_to_datum\n", + "from twinkle.server.model.tinker_datum import input_feature_to_datum\n", "from getpass import getpass" ] }, @@ -386,7 +386,7 @@ "metadata": {}, "outputs": [], "source": [ - "from twinkle_client import init_twinkle_client\n", + "from twinkle import init_twinkle_client\n", "from twinkle_client.model import MultiLoraTransformersModel\n", "# 步骤 1：初始化 Twinkle 客户端。\n", "# Tinker 检查点（twinkle:// 路径）由同一检查点服务解析\n", diff --git a/notebook/short_math_grpo.ipynb b/notebook/short_math_grpo.ipynb index fbe8ffc3..2d40fe91 100644 --- a/notebook/short_math_grpo.ipynb +++ b/notebook/short_math_grpo.ipynb @@ -136,7 +136,8 @@ "from typing import List, Tuple, Dict, Any\n", "\n", "from getpass import getpass\n", - "from twinkle import get_logger\nfrom twinkle_client import init_twinkle_client\n", + "from twinkle import get_logger\n", + "from twinkle import init_twinkle_client\n", "from twinkle.reward.base import Reward\n", "from twinkle.advantage import GRPOAdvantage\n", "from twinkle.dataset import DatasetMeta, Dataset\n", @@ -528,7 +529,7 @@ "metadata": {}, "outputs": [], "source": [ - "from twinkle_client import init_twinkle_client\n", + "from twinkle import init_twinkle_client\n", "from twinkle_client.model import MultiLoraTransformersModel\n", "# 步骤 1：初始化 Twinkle 客户端。\n", "# Tinker 检查点（twinkle:// 路径）由同一检查点服务解析\n", @@ -573,7 +574,8 @@ "source": [ "# 推理示例（使用线上服务，无需本地 GPU）\n", "from tinker import types\n", - "from twinkle import get_logger\nfrom twinkle_client import init_tinker_client\n", + "from twinkle import get_logger\n", + "from twinkle import init_tinker_client\n", "from twinkle.data_format import Message, Trajectory\n", "from twinkle.template import Template\n", "\n", diff --git a/src/twinkle/__init__.py b/src/twinkle/__init__.py index f80a4d74..af38ebfb 100644 --- a/src/twinkle/__init__.py +++ b/src/twinkle/__init__.py @@ -5,7 +5,7 @@ def init_tinker_client(**kwargs) -> None: - """Compatibility entry point; prefer ``twinkle_client.init_tinker_client``.""" + """Initialize the Tinker-compatible client without eager client imports.""" from twinkle_client import init_tinker_client as _init_tinker_client return _init_tinker_client(**kwargs) @@ -16,7 +16,7 @@ def init_twinkle_client( session_heartbeat_interval: int = 10, **kwargs, ) -> Any: - """Compatibility entry point; prefer ``twinkle_client.init_twinkle_client``.""" + """Initialize the Twinkle client without eager client imports.""" from twinkle_client import init_twinkle_client as _init_twinkle_client return _init_twinkle_client( base_url=base_url, diff --git a/src/twinkle/server/deployment.py b/src/twinkle/server/deployment.py index cdf2f868..45ca43fb 100644 --- a/src/twinkle/server/deployment.py +++ b/src/twinkle/server/deployment.py @@ -36,9 +36,10 @@ from ray import serve from typing import Any -from twinkle.protocol.types.errors import ErrorCategory, ErrorPayload +from twinkle.protocol.types.errors import ErrorCategory from twinkle.server.exceptions import TwinkleServerError from twinkle.server.middleware.auth import verify_request_token +from twinkle.server.task_errors import build_error_payload from twinkle.server.telemetry.http_middleware import create_metrics_middleware from twinkle.server.telemetry.tracing import create_tracing_middleware from twinkle.utils.logger import get_logger @@ -59,8 +60,8 @@ async def twinkle_server_error_handler(request: Request, exc: TwinkleServerError (``category=user``) carries no traceback. """ request_id = getattr(request.state, 'request_id', None) or '' - payload = ErrorPayload( - error=(str(exc) or exc.__class__.__name__), + payload = build_error_payload( + str(exc) or exc.__class__.__name__, category=exc.category, error_code=exc.error_code, request_id=request_id, @@ -120,8 +121,8 @@ async def validation_error_handler(request: Request, exc: RequestValidationError # server, so say so instead of leaving the caller to infer it from a field list. message += ('. Unknown fields are rejected; if this worked before, upgrade ' 'twinkle-kit on the client to match the server version.') - payload = ErrorPayload( - error=message[:1024], + payload = build_error_payload( + message, category=ErrorCategory.User, error_code=422, request_id=getattr(request.state, 'request_id', None) or '', @@ -241,12 +242,12 @@ async def catch_unhandled_exceptions(request: Request, call_next): # ``ErrorPayload`` body (Server category keeps the traceback) instead # of the legacy ``{'detail': }`` shape. request_id = getattr(request.state, 'request_id', None) or '' - payload = ErrorPayload( - error=(str(exc) or exc.__class__.__name__), + payload = build_error_payload( + str(exc) or exc.__class__.__name__, category=ErrorCategory.Server, error_code=500, request_id=request_id, - traceback=tb, + traceback_text=tb, ) return JSONResponse(status_code=500, content=payload.model_dump(mode='json', exclude_none=True)) diff --git a/src/twinkle/server/task_errors.py b/src/twinkle/server/task_errors.py index 13402611..cd94d7b7 100644 --- a/src/twinkle/server/task_errors.py +++ b/src/twinkle/server/task_errors.py @@ -19,7 +19,7 @@ def trim_traceback(text: str) -> str: return _TRUNCATION_MARKER + text[-keep:] -def task_error_payload( +def build_error_payload( error: str, *, request_id: str, @@ -27,14 +27,14 @@ def task_error_payload( category: ErrorCategory | str = ErrorCategory.Server, traceback_text: str | None = None, details: list[dict[str, Any]] | None = None, -) -> dict[str, Any]: - """Build a JSON-safe wire payload for direct or streaming responses.""" +) -> ErrorPayload: + """Build a validated error payload with bounded diagnostic text.""" if isinstance(category, str): category = ErrorCategory(category.lower()) tb = trim_traceback(traceback_text) if category is ErrorCategory.Server and traceback_text else None lines = str(error).splitlines() summary = (lines[0] if lines else '')[:_ERROR_MAX] - payload = ErrorPayload( + return ErrorPayload( error=summary, category=category, error_code=error_code, @@ -42,4 +42,24 @@ def task_error_payload( traceback=tb, details=details, ) + + +def task_error_payload( + error: str, + *, + request_id: str, + error_code: int = 500, + category: ErrorCategory | str = ErrorCategory.Server, + traceback_text: str | None = None, + details: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Build a JSON-safe wire payload for direct or streaming responses.""" + payload = build_error_payload( + error, + request_id=request_id, + error_code=error_code, + category=category, + traceback_text=traceback_text, + details=details, + ) return payload.model_dump(mode='json', exclude_none=True) diff --git a/src/twinkle_client/__init__.py b/src/twinkle_client/__init__.py index a3614d75..1177403b 100644 --- a/src/twinkle_client/__init__.py +++ b/src/twinkle_client/__init__.py @@ -24,7 +24,7 @@ def init_tinker_client(**kwargs) -> None: Example:: - >>> from twinkle_client import init_tinker_client + >>> from twinkle import init_tinker_client >>> init_tinker_client() >>> from tinker import ServiceClient >>> client = ServiceClient(base_url='http://localhost:8000', api_key='your_token') diff --git a/src/twinkle_client/skills/bundled/twinkle-training.md b/src/twinkle_client/skills/bundled/twinkle-training.md index 4010322a..281ae434 100644 --- a/src/twinkle_client/skills/bundled/twinkle-training.md +++ b/src/twinkle_client/skills/bundled/twinkle-training.md @@ -76,7 +76,7 @@ Large models: 2× TP for 32B, 4× TP for 72B ### 1. Initialization ```python -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client # Server Mode (primary — self-hosted) client = init_twinkle_client(base_url='http://localhost:8000', api_key='EMPTY_API_KEY') @@ -484,7 +484,7 @@ InputFeature( ```python import os from peft import LoraConfig -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.dataset import Dataset, DatasetMeta from twinkle.dataloader import DataLoader from twinkle.preprocessor import SelfCognitionProcessor @@ -559,7 +559,7 @@ rt.finish(status='completed') import gc from typing import List, Dict, Any from peft import LoraConfig -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.dataset import Dataset, DatasetMeta from twinkle.dataloader import DataLoader from twinkle.preprocessor import GSM8KProcessor @@ -671,7 +671,7 @@ import numpy as np import torch from typing import Any, Dict, List from peft import LoraConfig -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.dataset import Dataset, DatasetMeta from twinkle.dataloader import DataLoader from twinkle.preprocessor import EmojiDPOProcessor @@ -753,7 +753,7 @@ rt.finish(status='completed') import numpy as np import torch from peft import LoraConfig -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle.dataset import LazyDataset, DatasetMeta from twinkle.dataloader import DataLoader from twinkle.preprocessor import Preprocessor @@ -828,7 +828,7 @@ rt.finish(status='completed') ### Example 5: Sampling / Inference Only ```python -from twinkle_client import init_twinkle_client +from twinkle import init_twinkle_client from twinkle_client.sampler import vLLMSampler # 1. Init @@ -919,7 +919,7 @@ The TUI agent's `start_server` tool handles this automatically — generates con For GRPO with Tinker API: ```python -from twinkle_client import init_tinker_client +from twinkle import init_tinker_client init_tinker_client() from tinker import ServiceClient, types diff --git a/tests/server/integration/e2e_helpers.py b/tests/server/integration/e2e_helpers.py index ee56bf0d..2eca95f0 100644 --- a/tests/server/integration/e2e_helpers.py +++ b/tests/server/integration/e2e_helpers.py @@ -131,7 +131,7 @@ def create_grpo_dataset(data_slice=range(50)): def init_twinkle_client_session(): """Initialize the Twinkle client session.""" - from twinkle_client import init_twinkle_client + from twinkle import init_twinkle_client return init_twinkle_client(base_url=BASE_URL, api_key=API_KEY) @@ -205,7 +205,7 @@ def create_twinkle_sampler(): def init_tinker_client_session(): """Initialize the Tinker client session and return ServiceClient.""" - from twinkle_client import init_tinker_client + from twinkle import init_tinker_client init_tinker_client() from tinker import ServiceClient return ServiceClient(base_url=BASE_URL, api_key=API_KEY) diff --git a/tests/server/integration/test_full_cycle_e2e.py b/tests/server/integration/test_full_cycle_e2e.py index 2e5da2ff..6fa31f00 100644 --- a/tests/server/integration/test_full_cycle_e2e.py +++ b/tests/server/integration/test_full_cycle_e2e.py @@ -45,7 +45,7 @@ ) from twinkle import get_logger # noqa: E402 -from twinkle_client import init_twinkle_client # noqa: E402 +from twinkle import init_twinkle_client # noqa: E402 from twinkle.dataloader import DataLoader # noqa: E402 from twinkle.dataset import Dataset, DatasetMeta # noqa: E402 from twinkle_client.model import MultiLoraTransformersModel # noqa: E402 diff --git a/tests/server/integration/test_full_param_e2e.py b/tests/server/integration/test_full_param_e2e.py index 8f98bc61..857fbc71 100644 --- a/tests/server/integration/test_full_param_e2e.py +++ b/tests/server/integration/test_full_param_e2e.py @@ -42,7 +42,7 @@ ) from twinkle import get_logger # noqa: E402 -from twinkle_client import init_tinker_client # noqa: E402 +from twinkle import init_tinker_client # noqa: E402 from twinkle.dataloader import DataLoader # noqa: E402 from twinkle.dataset import Dataset, DatasetMeta # noqa: E402 from twinkle.preprocessor import SelfCognitionProcessor # noqa: E402 diff --git a/tests/server/integration/test_mock_mode_startup.py b/tests/server/integration/test_mock_mode_startup.py index f1838b2c..d5f70606 100644 --- a/tests/server/integration/test_mock_mode_startup.py +++ b/tests/server/integration/test_mock_mode_startup.py @@ -211,7 +211,7 @@ def test_mock_mode_reaches_ready_under_30s_and_is_deterministic(ray_cluster) -> def _exercise_twinkle_clients(base: str) -> None: - from twinkle_client import init_twinkle_client + from twinkle import init_twinkle_client from twinkle_client.model import MultiLoraTransformersModel from twinkle_client.sampler import vLLMSampler @@ -308,7 +308,7 @@ def _exercise_tinker_client(base: str) -> None: import os from tinker import ServiceClient, types - from twinkle_client import init_tinker_client + from twinkle import init_tinker_client # patch_tinker injects Twinkle's auth + Ray Serve multiplex headers and # lifts tinker's ``tml-`` api-key prefix check so EMPTY_TOKEN passes. diff --git a/tests/server/integration/test_nccl_safe_tinker_e2e.py b/tests/server/integration/test_nccl_safe_tinker_e2e.py index 6b8bc3e0..23259c7c 100644 --- a/tests/server/integration/test_nccl_safe_tinker_e2e.py +++ b/tests/server/integration/test_nccl_safe_tinker_e2e.py @@ -41,7 +41,7 @@ def _init_client(): os.environ['TINKER_BASE_URL'] = SERVER_URL os.environ['TWINKLE_SERVER_TOKEN'] = 'EMPTY_TOKEN' - from twinkle_client import init_tinker_client + from twinkle import init_tinker_client init_tinker_client() from tinker import ServiceClient return ServiceClient().create_lora_training_client(base_model=BASE_MODEL, rank=16) diff --git a/tests/server/integration/test_nccl_safe_twinkle_e2e.py b/tests/server/integration/test_nccl_safe_twinkle_e2e.py index 5effc140..705e2ec3 100644 --- a/tests/server/integration/test_nccl_safe_twinkle_e2e.py +++ b/tests/server/integration/test_nccl_safe_twinkle_e2e.py @@ -40,7 +40,7 @@ def _init_client(): from peft import LoraConfig - from twinkle_client import init_twinkle_client + from twinkle import init_twinkle_client from twinkle_client.model import MultiLoraTransformersModel init_twinkle_client(base_url=SERVER_URL, api_key='EMPTY_TOKEN') diff --git a/tests/server/lifecycle/test_preflight_rejection.py b/tests/server/lifecycle/test_preflight_rejection.py index 0c481ee8..74639c13 100644 --- a/tests/server/lifecycle/test_preflight_rejection.py +++ b/tests/server/lifecycle/test_preflight_rejection.py @@ -113,3 +113,20 @@ async def boom(request: Request): assert body['error'] == 'nope' assert body['category'] == 'user' assert body['error_code'] == 409 + + +def test_error_handler_bounds_overlong_domain_error(): + app = FastAPI() + app.add_exception_handler(TwinkleServerError, twinkle_server_error_handler) + + @app.get('/boom') + async def boom(request: Request): + raise RequestRejectedError(f'bad request {"X" * 2048}\ninternal detail', error_code=409) + + response = TestClient(app, raise_server_exceptions=False).get('/boom') + assert response.status_code == 409 + body = response.json() + assert len(body['error']) == 1024 + assert '\n' not in body['error'] + assert body['category'] == 'user' + assert 'traceback' not in body diff --git a/tests/server/state/test_error_payload.py b/tests/server/state/test_error_payload.py index 7b251328..417470e1 100644 --- a/tests/server/state/test_error_payload.py +++ b/tests/server/state/test_error_payload.py @@ -5,7 +5,7 @@ import pytest from pydantic import ValidationError -from twinkle.server.task_errors import task_error_payload +from twinkle.server.task_errors import build_error_payload, task_error_payload from twinkle.protocol.types.errors import ErrorCategory, ErrorPayload @@ -35,6 +35,35 @@ def test_task_error_payload_shapes_and_sanitizes_errors(): } +def test_build_error_payload_bounds_text_and_preserves_metadata(): + details = [{'field': 'adapter_name'}] + payload = build_error_payload( + f'bad input {"X" * 2048}\nignored second line', + request_id='req_build', + error_code=422, + category='user', + traceback_text='server stack', + details=details, + ) + + assert isinstance(payload, ErrorPayload) + assert len(payload.error) == 1024 + assert '\n' not in payload.error + assert payload.category is ErrorCategory.User + assert payload.error_code == 422 + assert payload.request_id == 'req_build' + assert payload.traceback is None + assert payload.details == details + assert task_error_payload( + f'bad input {"X" * 2048}\nignored second line', + request_id='req_build', + error_code=422, + category='user', + traceback_text='server stack', + details=details, + ) == payload.model_dump(mode='json', exclude_none=True) + + def test_user_category_carries_no_traceback(): payload = task_error_payload( 'invalid field', request_id='req_2', error_code=422, diff --git a/tests/server/test_deployment_exception_boundary.py b/tests/server/test_deployment_exception_boundary.py index 1e9c9fe0..0ed0c63c 100644 --- a/tests/server/test_deployment_exception_boundary.py +++ b/tests/server/test_deployment_exception_boundary.py @@ -46,3 +46,33 @@ async def boom(): response = client.get('/healthz') assert response.status_code == 200 assert response.json() == {'ok': True, 'health_calls': 1} + + +def test_deployment_app_bounds_overlong_unhandled_error(monkeypatch): + + class _ReplicaId: + unique_id = 'replica-test' + + class _Context: + replica_id = _ReplicaId() + + from twinkle.server import deployment + + monkeypatch.setattr(deployment.serve, 'get_replica_context', lambda: _Context()) + + def register_routes(app: FastAPI, _get_self): + + @app.get('/boom') + async def boom(): + raise RuntimeError(f'first line {"X" * 2048}\nsecond line') + + client = TestClient(build_deployment_app('Test', register_routes)) + response = client.get('/boom', headers={'x-request-id': 'long-error'}) + + assert response.status_code == 500 + body = response.json() + assert len(body['error']) == 1024 + assert '\n' not in body['error'] + assert len(body['traceback']) <= 65536 + assert body['traceback'].endswith('second line\n') + assert body['request_id'] == 'long-error' diff --git a/tests/server/validation/test_request_wire.py b/tests/server/validation/test_request_wire.py index 05e9bb5e..73759d25 100644 --- a/tests/server/validation/test_request_wire.py +++ b/tests/server/validation/test_request_wire.py @@ -82,6 +82,18 @@ def test_the_error_names_the_client_version_mismatch(client): assert 'upgrade' in response.json()['error'].lower() +def test_overlong_validation_summary_is_bounded_without_losing_details(client): + field = 'unknown_' + ('X' * 2048) + response = _post(client, '/forward', _valid_forward(**{field: 1})) + + assert response.status_code == 422 + body = response.json() + assert len(body['error']) == 1024 + assert body['category'] == 'user' + assert body['details'][0]['field'] == field + assert 'traceback' not in body + + def test_the_error_body_is_an_error_payload_not_fastapi_detail(client): """One error shape on the wire, or a client has to learn two.""" body = _post(client, '/forward', _valid_forward(unknown=1)).json() From b4776b7f8fe440444560f5017329722e583b8ab9 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 21 Sep 2026 12:38:44 +0000 Subject: [PATCH 31/32] fix(client): resolve PR review feedback --- src/twinkle/protocol/types/model.py | 8 +++--- src/twinkle_client/auto/agent/core.py | 5 +++- src/twinkle_client/auto/agent/monitor.py | 5 +++- .../model/multi_lora_transformers.py | 25 +++++++++++++++---- tests/data_format/test_tq_fields.py | 5 ++-- tests/twinkle_client/test_async_components.py | 18 +++++++++++++ tests/twinkle_client/test_request_builder.py | 5 ++++ 7 files changed, 58 insertions(+), 13 deletions(-) diff --git a/src/twinkle/protocol/types/model.py b/src/twinkle/protocol/types/model.py index b22d3164..f2e6bd97 100644 --- a/src/twinkle/protocol/types/model.py +++ b/src/twinkle/protocol/types/model.py @@ -134,12 +134,12 @@ class ForwardRequest(_InlineForwardBase): class ForwardOnlyRequest(_InlineForwardBase): """Body of ``POST /twinkle/forward_only``: no graph, no gradients. - ``adapter_name`` is optional here -- a reference forward may run against the base - weights -- and there is no ``seq_id`` because nothing is mutated to be idempotent - about. + An existing ``adapter_name`` supplies the template and adapter context, even when + ``disable_lora`` requests base-weight inference. There is no ``seq_id`` because + nothing is mutated to be idempotent about. """ - adapter_name: str | None = None + adapter_name: str disable_lora: bool | None = backend_kwarg(default=None) diff --git a/src/twinkle_client/auto/agent/core.py b/src/twinkle_client/auto/agent/core.py index d2c4056c..88c387f3 100644 --- a/src/twinkle_client/auto/agent/core.py +++ b/src/twinkle_client/auto/agent/core.py @@ -5,7 +5,7 @@ import asyncio import json -from typing import Any, Callable +from typing import TYPE_CHECKING, Any, Callable from twinkle.utils.logger import get_logger from twinkle_client.auto.agent.prompts import SYSTEM_PROMPT @@ -14,6 +14,9 @@ logger = get_logger() +if TYPE_CHECKING: + from openai import AsyncOpenAI + class AgentLoop: """Async tool-calling agent loop using OpenAI-compatible API. diff --git a/src/twinkle_client/auto/agent/monitor.py b/src/twinkle_client/auto/agent/monitor.py index 1987f076..34f2f415 100644 --- a/src/twinkle_client/auto/agent/monitor.py +++ b/src/twinkle_client/auto/agent/monitor.py @@ -19,13 +19,16 @@ import re import time from pathlib import Path -from typing import Any, Callable +from typing import TYPE_CHECKING, Any, Callable from twinkle.utils.logger import get_logger from twinkle_client.auto.connection import LocalConnection logger = get_logger() +if TYPE_CHECKING: + from openai import AsyncOpenAI + # Maximum auto-fix attempts per run (prevent infinite retry loops) _MAX_FIX_ATTEMPTS = 3 diff --git a/src/twinkle_client/model/multi_lora_transformers.py b/src/twinkle_client/model/multi_lora_transformers.py index e5617737..a9fbfd8e 100644 --- a/src/twinkle_client/model/multi_lora_transformers.py +++ b/src/twinkle_client/model/multi_lora_transformers.py @@ -1,8 +1,14 @@ +from __future__ import annotations + import itertools import logging import threading +from collections.abc import Mapping from pathlib import Path -from typing import Any, Dict, Optional +from typing import TYPE_CHECKING, Any, Dict, Optional + +if TYPE_CHECKING: + from peft import LoraConfig from twinkle.protocol.types import model as model_types from twinkle.protocol.types.component import DataRef @@ -107,13 +113,22 @@ def _next_seq_id(self) -> int: # Adapter lifecycle # ------------------------------------------------------------------ # - def add_adapter_to_model(self, adapter_name: str, config: Optional[Dict[str, Any]] = None, **kwargs) -> None: + def add_adapter_to_model( + self, + adapter_name: str, + config: LoraConfig | Mapping[str, Any] | None = None, + **kwargs, + ) -> None: """Add a new adapter to the model. Pass a peft ``LoraConfig`` (or its dict form) for LoRA training against a LoRA-mode deployment. Pass ``config=None`` for full-parameter training against a ``train_mode: full`` deployment. """ + if isinstance(config, Mapping): + from peft import LoraConfig + + config = LoraConfig(**config) save_dir = kwargs.pop('save_dir', None) if save_dir: save_dir = Path(save_dir).expanduser().resolve().as_posix() @@ -385,7 +400,7 @@ def set_processor(self, processor_cls: str, **kwargs) -> None: adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) - def add_metric(self, metric_cls: str, is_training: Optional[bool] = None, **kwargs) -> None: + def add_metric(self, metric_cls: str, is_training: bool | None = None, **kwargs) -> None: """Add a metric to the model.""" self._submit( 'add_metric', @@ -440,7 +455,7 @@ def load(self, name: str, **kwargs) -> None: adapter_name=kwargs.pop('adapter_name', self.adapter_name), **kwargs) - def resume_from_checkpoint(self, name: str, *, resume_only_model: bool = False, **kwargs) -> Dict[str, Any]: + def resume_from_checkpoint(self, name: str, *, resume_only_model: bool = False, **kwargs) -> dict[str, Any]: """Resume weights (and optionally optimizer state) from a checkpoint.""" progress = self._submit( 'resume_from_checkpoint', @@ -456,7 +471,7 @@ def upload_to_hub( self, checkpoint_dir: str, hub_model_id: str, - hub_token: Optional[str] = None, + hub_token: str | None = None, async_upload: bool = True, poll_interval: float = 5.0, ) -> None: diff --git a/tests/data_format/test_tq_fields.py b/tests/data_format/test_tq_fields.py index 3d135548..e882f65c 100644 --- a/tests/data_format/test_tq_fields.py +++ b/tests/data_format/test_tq_fields.py @@ -12,10 +12,9 @@ import pytest -tensordict = pytest.importorskip('tensordict') - def test_rows_to_tq_fields_empty_rows(): + pytest.importorskip('tensordict') from twinkle.data_format import rows_to_tq_fields packed = rows_to_tq_fields([]) @@ -24,6 +23,7 @@ def test_rows_to_tq_fields_empty_rows(): def test_rows_to_tq_fields_rejects_inconsistent_fields(): """Rows with differing key sets must raise, not silently pack a ragged TensorDict.""" + pytest.importorskip('tensordict') from twinkle.data_format import rows_to_tq_fields with pytest.raises(ValueError): @@ -32,6 +32,7 @@ def test_rows_to_tq_fields_rejects_inconsistent_fields(): def test_columns_to_tq_fields_mixes_numeric_and_non_numeric(): """Numeric columns go through ``torch.tensor``; the rest through ``NonTensorStack``.""" + pytest.importorskip('tensordict') import torch from twinkle.data_format import columns_to_tq_fields diff --git a/tests/twinkle_client/test_async_components.py b/tests/twinkle_client/test_async_components.py index 875ddc17..0a092627 100644 --- a/tests/twinkle_client/test_async_components.py +++ b/tests/twinkle_client/test_async_components.py @@ -110,6 +110,24 @@ def test_model_inline_forward_methods_keep_the_original_endpoints(monkeypatch) - assert calls[-1][1]['micro_batch_size'] == 1 +def test_model_add_adapter_serializes_dict_lora_config(monkeypatch) -> None: + """Dict input remains compatible with the strict serialized LoRA wire field.""" + from twinkle_client.model import multi_lora_transformers as module + + calls: list = [] + _patch_transport(monkeypatch, calls, + lambda url: {} if url.endswith('/create') else _completed({'status': 'ok'})) + + model = module.MultiLoraTransformersModel('ms://base') + model.add_adapter_to_model('adapter', {'r': 4, 'target_modules': 'all-linear'}) + + url, body = calls[-1] + assert url.endswith('/model/base/twinkle/add_adapter_to_model') + assert isinstance(body['config'], str) + assert 'LoraConfig' in body['config'] + assert model.adapter_name == 'adapter' + + def test_undeclared_forward_arguments_are_routed_to_loss_kwargs(monkeypatch) -> None: """A loss input is not a declared field, so it travels in the passthrough region. diff --git a/tests/twinkle_client/test_request_builder.py b/tests/twinkle_client/test_request_builder.py index 9617b04b..0c7a15bc 100644 --- a/tests/twinkle_client/test_request_builder.py +++ b/tests/twinkle_client/test_request_builder.py @@ -128,6 +128,11 @@ def test_a_missing_required_field_fails_in_process(): build_request(model_types.ForwardRequest, inputs=[{'input_ids': [1]}]) +def test_forward_only_requires_an_adapter_context(): + with pytest.raises(ValidationError): + build_request(model_types.ForwardOnlyRequest, inputs=[{'input_ids': [1]}]) + + # --------------------------------------------------------------------------- # # Serialization # --------------------------------------------------------------------------- # From fa744cedf430112c9c10997b89c1aeccd0fc5751 Mon Sep 17 00:00:00 2001 From: Yunnglin Date: Mon, 21 Sep 2026 13:45:55 +0000 Subject: [PATCH 32/32] style: simplify spec-tracking comments --- src/twinkle/model/transformers/moe/expert_parallel.py | 2 +- src/twinkle/server/checkpoint/__init__.py | 2 +- src/twinkle/server/checkpoint/checkpoint_manager.py | 2 -- src/twinkle/server/checkpoint/models.py | 2 -- src/twinkle/server/checkpoint/paths.py | 2 -- src/twinkle/server/checkpoint/training_run_manager.py | 2 -- src/twinkle/server/config/backend_dispatch.py | 2 +- src/twinkle/server/launcher/builder_registry.py | 3 --- src/twinkle/server/launcher/env_propagation.py | 5 ++--- src/twinkle/server/middleware/__init__.py | 2 +- src/twinkle_agentic/preprocessor/intent_classifier.py | 8 ++++---- tests/server/contract/test_client_api_contract.py | 3 +-- tests/twinkle_client/test_error_parsing.py | 2 +- tests/twinkle_client/test_future_layer.py | 2 +- 14 files changed, 13 insertions(+), 26 deletions(-) diff --git a/src/twinkle/model/transformers/moe/expert_parallel.py b/src/twinkle/model/transformers/moe/expert_parallel.py index 218e7b33..46717a8c 100644 --- a/src/twinkle/model/transformers/moe/expert_parallel.py +++ b/src/twinkle/model/transformers/moe/expert_parallel.py @@ -234,7 +234,7 @@ def forward(hidden_states: torch.Tensor, *args, **kwargs): else: raise ValueError(f'Unsupported hidden_states ndim: {hidden_states.ndim}') - # R2 / R3 routing replay: pass block-level replay state + # Pass block-level routing replay state. from .router_replay import get_replay_state replay_state = get_replay_state(block_name) diff --git a/src/twinkle/server/checkpoint/__init__.py b/src/twinkle/server/checkpoint/__init__.py index e6a129aa..47308367 100644 --- a/src/twinkle/server/checkpoint/__init__.py +++ b/src/twinkle/server/checkpoint/__init__.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Checkpoint subsystem (TIER 2 consolidation). +"""Checkpoint subsystem. Top-level package consolidating the checkpoint base classes (split from the former 1017-line ``utils/checkpoint_base.py``, now deleted) with the concrete diff --git a/src/twinkle/server/checkpoint/checkpoint_manager.py b/src/twinkle/server/checkpoint/checkpoint_manager.py index 773494b6..8b021008 100644 --- a/src/twinkle/server/checkpoint/checkpoint_manager.py +++ b/src/twinkle/server/checkpoint/checkpoint_manager.py @@ -1,7 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Abstract base checkpoint manager. - -Relocated from ``utils/checkpoint_base.py`` (TIER 2 consolidation). No logic change. """ from __future__ import annotations diff --git a/src/twinkle/server/checkpoint/models.py b/src/twinkle/server/checkpoint/models.py index fdb885c2..fa42bc28 100644 --- a/src/twinkle/server/checkpoint/models.py +++ b/src/twinkle/server/checkpoint/models.py @@ -1,8 +1,6 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Internal Pydantic base specs used as type constraints for the generic checkpoint / training-run managers. - -Relocated from ``utils/checkpoint_base.py`` (TIER 2 consolidation). No logic change. """ from __future__ import annotations diff --git a/src/twinkle/server/checkpoint/paths.py b/src/twinkle/server/checkpoint/paths.py index 202f1747..d22d4c41 100644 --- a/src/twinkle/server/checkpoint/paths.py +++ b/src/twinkle/server/checkpoint/paths.py @@ -1,8 +1,6 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Path constants, token hashing, client-save-dir resolution, and permission helpers for the checkpoint subsystem. - -Relocated from ``utils/checkpoint_base.py`` (TIER 2 consolidation). No logic change. """ from __future__ import annotations diff --git a/src/twinkle/server/checkpoint/training_run_manager.py b/src/twinkle/server/checkpoint/training_run_manager.py index 6526fb86..7075268e 100644 --- a/src/twinkle/server/checkpoint/training_run_manager.py +++ b/src/twinkle/server/checkpoint/training_run_manager.py @@ -1,7 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Base file manager and abstract training-run manager. - -Relocated from ``utils/checkpoint_base.py`` (TIER 2 consolidation). No logic change. """ from __future__ import annotations diff --git a/src/twinkle/server/config/backend_dispatch.py b/src/twinkle/server/config/backend_dispatch.py index 65613e57..b4da33f9 100644 --- a/src/twinkle/server/config/backend_dispatch.py +++ b/src/twinkle/server/config/backend_dispatch.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Generic validate-then-dispatch helper for backend selectors (TIER 1, R12). +"""Generic validate-then-dispatch helper for backend selectors. The Model backend selector (``mock | transformers | megatron``) and the Sampler type selector (``mock | vllm | torch``) share one validate-then-dispatch shape. diff --git a/src/twinkle/server/launcher/builder_registry.py b/src/twinkle/server/launcher/builder_registry.py index a18be38b..9c81d07e 100644 --- a/src/twinkle/server/launcher/builder_registry.py +++ b/src/twinkle/server/launcher/builder_registry.py @@ -1,9 +1,6 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """``import_path`` → deployment-builder resolution. -Extracted from the former single-file ``launcher.py`` (TIER 3 same-named-package -decomposition). No logic change. - The operator-facing YAML ``import_path`` literals (``"server"``, ``"model"``, ``"sampler"``, ``"processor"``, ``"data_plane"``) resolve to internal builder functions. The function selected by the ``"server"`` literal was renamed diff --git a/src/twinkle/server/launcher/env_propagation.py b/src/twinkle/server/launcher/env_propagation.py index 2b333aba..c478444d 100644 --- a/src/twinkle/server/launcher/env_propagation.py +++ b/src/twinkle/server/launcher/env_propagation.py @@ -1,9 +1,8 @@ # Copyright (c) ModelScope Contributors. All rights reserved. """Collection of telemetry / persistence env vars for propagation to Ray workers. -Extracted from the former single-file ``launcher.py`` (TIER 3 same-named-package -decomposition). No logic change. These vars are read inside each Ray Serve -worker process — telemetry by ``ensure_telemetry_initialized()`` and persistence +These variables are read inside each Ray Serve worker process — telemetry by +``ensure_telemetry_initialized()`` and persistence by ``PersistenceConfig.from_env()`` — so the chosen backend / telemetry config is independent of deployment startup order. """ diff --git a/src/twinkle/server/middleware/__init__.py b/src/twinkle/server/middleware/__init__.py index b1f0e5b0..9bb98c0a 100644 --- a/src/twinkle/server/middleware/__init__.py +++ b/src/twinkle/server/middleware/__init__.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""HTTP middleware for the server deployments (moved here from ``server/utils/`` by R14). +"""HTTP middleware for the server deployments. Sits next to ``deployment.py``'s middleware stack: ``auth.verify_request_token`` is the token-verification middleware registered by ``build_deployment_app``. diff --git a/src/twinkle_agentic/preprocessor/intent_classifier.py b/src/twinkle_agentic/preprocessor/intent_classifier.py index 6d1b21c2..35e1c10f 100644 --- a/src/twinkle_agentic/preprocessor/intent_classifier.py +++ b/src/twinkle_agentic/preprocessor/intent_classifier.py @@ -39,7 +39,7 @@ r'\\times|\\div|\\pm|\\leq|\\geq|\\neq|\\approx|\\equiv|' r'\\infty|\\pi|\\alpha|\\beta|\\gamma|\\theta|\\lambda|\\mu|\\sigma|\\prod|\\to|\\rightarrow|' r'\\\[.+?\\\]|' - # R1-distill writes math in plain Unicode without $...$; catch operators, Greek, sub/super digits, fractions. + # Also match plain-Unicode operators, Greek letters, super/subscripts, and fractions. r'[×÷±°∑∏∫√∂∇∞∈∋⊂⊃⊆⊇≤≥≠≈≡≅∝⇒⇔]|' r'[α-ωΔΘΛΞΠΣΦΨΩ]|' r'[⁰¹²³⁴-⁹₀-₉]|' @@ -355,8 +355,8 @@ class IntentClassifier(Preprocessor): Pure-heuristic, no LLM. Each intent is a pluggable :class:`IntentDetector`; pass ``detectors=[...]`` to extend or override. - R3: this is an *annotator* — by default it never drops rows - (``drop_no_key_rounds=False``); rows with no detected key round are simply + This is an *annotator*: by default it never drops rows + (``drop_no_key_rounds=False``), and rows with no detected key round are simply tagged ``INTENT_OTHER``. Set ``drop_no_key_rounds=True`` to also filter. Annotates per row:: @@ -366,7 +366,7 @@ class IntentClassifier(Preprocessor): ('intents', dict[str, str])] # per-round intent """ - # R4: default to the detectors with a live downstream consumer. The heavier + # Default to detectors with a live downstream consumer. The heavier # heuristics (ComplexLogic / Reasoning / UserDissatisfaction) are kept as # importable classes but dropped from the default set — their outputs had no # active consumer. Pass ``detectors=[...]`` to re-enable them. diff --git a/tests/server/contract/test_client_api_contract.py b/tests/server/contract/test_client_api_contract.py index 3e835ad4..f543ee43 100644 --- a/tests/server/contract/test_client_api_contract.py +++ b/tests/server/contract/test_client_api_contract.py @@ -53,8 +53,7 @@ def test_full_surface_extraction_is_self_consistent(): def test_schedule_task_and_wait_removed(): - # server-request-lifecycle (Part 2) deletes the in-process blocking wait: the - # future record is now the single delivery channel. + # Future records replace the in-process blocking wait. from twinkle.server.task_queue.mixin import TaskQueueMixin assert not hasattr(TaskQueueMixin, 'schedule_task_and_wait') assert hasattr(TaskQueueMixin, 'submit_and_peek') diff --git a/tests/twinkle_client/test_error_parsing.py b/tests/twinkle_client/test_error_parsing.py index f55fd1ed..037fd0db 100644 --- a/tests/twinkle_client/test_error_parsing.py +++ b/tests/twinkle_client/test_error_parsing.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Client error-response parsing ( / Requirement 3 #7-#11).""" +"""Client error-response parsing tests.""" from __future__ import annotations import pytest diff --git a/tests/twinkle_client/test_future_layer.py b/tests/twinkle_client/test_future_layer.py index b33ad137..e078b538 100644 --- a/tests/twinkle_client/test_future_layer.py +++ b/tests/twinkle_client/test_future_layer.py @@ -1,5 +1,5 @@ # Copyright (c) ModelScope Contributors. All rights reserved. -"""Client_Future_Layer unit tests ( / Requirement 4). +"""Client future-layer unit tests. ``resolve`` is exercised against fabricated envelopes and a monkeypatched ``_post_retrieve``; no server or network is involved.