From 935d23606b0419f687ecd87e5ef454aabf75e0d6 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Mon, 6 Jul 2026 19:44:34 +0000 Subject: [PATCH 01/28] feat(dev): add local vLLM model helper Signed-off-by: Aaron Gonzales --- .gitignore | 1 + README.md | 15 + docs/concepts/local-vllm.md | 131 ++ docs/concepts/models.md | 2 + mkdocs.yml | 1 + pyproject.toml | 3 + tests/tools/test_vllm_debug.py | 184 +++ tools/vllm_debug.py | 290 ++++ uv.lock | 2628 ++++++++++++++++++++++++++++++-- 9 files changed, 3168 insertions(+), 87 deletions(-) create mode 100644 docs/concepts/local-vllm.md create mode 100644 tests/tools/test_vllm_debug.py create mode 100644 tools/vllm_debug.py diff --git a/.gitignore b/.gitignore index a5bc84b5..deef06d2 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ uv.lock.bak # Environments .env .env.* +.mise.local.toml env/ venv/ ENV/ diff --git a/README.md b/README.md index 28ff3144..5afba45c 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,21 @@ anonymizer --help # CLI usage make install-pre-commit # Install pre-commit hooks ``` +### Local vLLM debugging + +On a Linux GPU host, install the optional local-model dependency group and use +`tools/vllm_debug.py` to start or probe an OpenAI-compatible vLLM server: + +```bash +uv sync --group dev --group local-models +uv run python tools/vllm_debug.py models --cached --json +uv run python tools/vllm_debug.py serve /path/to/cached/snapshot --served-model-name anonymizer-local +uv run python tools/vllm_debug.py models --endpoint http://127.0.0.1:8000/v1 +``` + +The helper does not prefetch model weights. Use `--vllm-python /path/to/python` +when vLLM is installed in another virtual environment. The [local vLLM guide](docs/concepts/local-vllm.md) covers GPU-host requirements, Anonymizer configuration, verification, and internal-network access. + --- ## Requirements diff --git a/docs/concepts/local-vllm.md b/docs/concepts/local-vllm.md new file mode 100644 index 00000000..884ed208 --- /dev/null +++ b/docs/concepts/local-vllm.md @@ -0,0 +1,131 @@ + + + +# Run Local vLLM Models + +This guide is for contributors who operate an Anonymizer source checkout on a +Linux host with NVIDIA GPUs. It starts a local, OpenAI-compatible [vLLM](https://docs.vllm.ai/) +endpoint for development, evaluation, or an internal deployment. It does not +download a model or configure production networking for you. + +The helper script, [`tools/vllm_debug.py`](https://github.com/NVIDIA-NeMo/Anonymizer/blob/main/tools/vllm_debug.py), is a source-tree development tool. It is not included in the `nemo-anonymizer` package. + +## Host Requirements + +Before installing the local-model dependencies, confirm that the host has: + +- Linux, Python 3.11 or later, and an NVIDIA GPU that has enough free VRAM for the selected model and its context length. +- An NVIDIA driver that can communicate with the GPU. Verify this with `nvidia-smi`. +- Access to the model weights, either through the Hugging Face cache or a local model directory. Observe the model license and access controls. + +The dependency group installs vLLM and its CUDA-compatible dependencies. Driver, GPU, model-size, and CUDA compatibility remain properties of the host. Start with a small model and a short context length before adopting a model for a benchmark or workflow. + +## Install the Local-Model Environment + +From a source checkout, install the development and local-model groups: + +```bash +uv sync --group dev --group local-models +nvidia-smi +uv run python -c 'import torch; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))' +``` + +If vLLM is kept in a separate virtual environment, the helper can use that interpreter with `--vllm-python /path/to/python`. This is useful when the model-serving environment has different CUDA constraints from the Anonymizer development environment. + +## Select a Cached Model + +The helper discovers snapshots already in the Hugging Face hub cache. It does not fetch model weights: + +```bash +uv run python tools/vllm_debug.py models --cached --json +``` + +Pass a `snapshot_path` from that output to `serve`. A Hugging Face model ID is also accepted by vLLM, but may trigger a download. + +## Start and Verify a Server + +Run the server from a terminal that will remain open: + +```bash +uv run python tools/vllm_debug.py serve /path/to/cached/snapshot \ + --served-model-name anonymizer-local \ + --gpu-memory-utilization 0.85 \ + --max-model-len 8192 +``` + +The default bind address, `127.0.0.1:8000`, keeps the endpoint local to the GPU host. `--served-model-name` gives clients a stable model ID, independent of the cache directory name. Stop the server with `Ctrl-C`. + +From another terminal, verify both the model registration and one completion: + +```bash +uv run python tools/vllm_debug.py models +uv run python tools/vllm_debug.py call \ + --model anonymizer-local \ + --prompt 'Reply with the word ready.' \ + --timeout-seconds 120 +``` + +Use `--dry-run` with `serve` to print the vLLM command before reserving GPU memory. For multi-GPU models, add `--tensor-parallel-size N`. For a LoRA adapter, add `--adapter /path/to/adapter` and, if needed, `--adapter-name NAME`. + +## Connect Anonymizer + +vLLM presents an OpenAI-compatible endpoint. Add it to a custom provider file: + +```yaml title="providers.yaml" +providers: + - name: local-vllm + endpoint: http://127.0.0.1:8000/v1 + provider_type: openai + api_key: EMPTY # vLLM has no API key unless one is configured below +``` + +In a custom `models.yaml`, set each local model configuration's `provider` to `local-vllm` and its `model` to the server's served-model name, `anonymizer-local` in the example above. Then pass both files to `Anonymizer`: + +```python +from anonymizer import Anonymizer + +anonymizer = Anonymizer( + model_providers="providers.yaml", + model_configs="models.yaml", +) +``` + +`model_configs` replaces Anonymizer's entire bundled model pool. Copy the bundled [`models.yaml`](https://github.com/NVIDIA-NeMo/Anonymizer/blob/main/src/anonymizer/config/default_model_configs/models.yaml), retain every alias required by the roles you use, and change only the models that should route to vLLM. See [Custom models](models.md#custom-models) for the role map and validation command. + +Use `anonymizer.validate_config(config)` before processing data. A successful HTTP probe only confirms that the server responds. It does not establish that a model is suitable for detection, replacement, or privacy-preserving rewrite quality. + +## Access From Another Host + +Keep the default loopback binding whenever Anonymizer and vLLM run on the same machine. If a trusted internal client needs the endpoint, use a network policy or firewall as well as a server API key: + +```bash +# .env.local is ignored by this repository. Do not commit it. +LOCAL_VLLM_API_KEY='replace-with-a-long-random-secret' + +# Load it in the shell that starts the server and the client. +set -a; source .env.local; set +a + +uv run python tools/vllm_debug.py serve /path/to/cached/snapshot \ + --host 0.0.0.0 \ + --served-model-name anonymizer-local \ + --api-key-env LOCAL_VLLM_API_KEY +``` + +Set the same secret's environment-variable name in the Anonymizer provider configuration: + +```yaml title="providers.yaml" +providers: + - name: local-vllm + endpoint: http://gpu-host.internal:8000/v1 + provider_type: openai + api_key: LOCAL_VLLM_API_KEY +``` + +Use an ignored `.env.local` file, or an ignored `.mise.local.toml` file when your checkout uses Mise, to keep local endpoint credentials out of version control. Load the secret only in the shell or task runner that starts the server and client. Do not expose a raw vLLM endpoint to the public internet. For production access, place it behind your organization's authenticated TLS-enabled network boundary. + +## Operating Notes + +- List the server's registered model IDs with `models` after every model or adapter change. Client `model` values must match one of those IDs. +- Start conservatively with `--gpu-memory-utilization` and `--max-model-len`; increase them only after observing stable GPU memory use and latency. +- Keep GLiNER separate from the LLM when GPU memory is constrained. The [self-hosted GLiNER guide](self-hosting-gliner.md) describes the detection endpoint. +- Treat local-model output as untrusted until it has passed the same Anonymizer preview, evaluation, and privacy review used for any other provider. diff --git a/docs/concepts/models.md b/docs/concepts/models.md index f7646e7f..cb92eec0 100644 --- a/docs/concepts/models.md +++ b/docs/concepts/models.md @@ -36,6 +36,8 @@ Each pipeline stage has a **role** mapped to one of these aliases. See the full Pass `model_providers` when you need a non-default endpoint — for example OpenAI, OpenRouter, a local GLiNER server, or an internal inference deployment. Plain `Anonymizer()` already uses bundled [build.nvidia.com](https://build.nvidia.com) settings; override only when your models point at a different provider name or URL. +For a GPU-hosted OpenAI-compatible LLM endpoint, see [Run local vLLM models](local-vllm.md). That guide covers the source-tree helper, optional dependencies, and a `local-vllm` provider configuration. + Set your API keys first: ```bash diff --git a/mkdocs.yml b/mkdocs.yml index f75c218a..1d047a8d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -161,6 +161,7 @@ nav: - Choosing a Strategy: concepts/choosing-a-strategy.md - Evaluation: concepts/evaluation.md - Self-hosting GLiNER: concepts/self-hosting-gliner.md + - Run Local vLLM Models: concepts/local-vllm.md - Troubleshooting: troubleshooting.md - Tutorials: - Overview: tutorials/index.md diff --git a/pyproject.toml b/pyproject.toml index 4c5c4b82..9c51f894 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,9 @@ notebooks = [ "jupytext>=1.16.0,<2", "pillow>=12.0.0,<13", ] +local-models = [ + "vllm==0.20.0; sys_platform == 'linux'", +] [build-system] requires = ["hatchling", "uv-dynamic-versioning"] diff --git a/tests/tools/test_vllm_debug.py b/tests/tools/test_vllm_debug.py new file mode 100644 index 00000000..5ab66f25 --- /dev/null +++ b/tests/tools/test_vllm_debug.py @@ -0,0 +1,184 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Behavior tests for the local vLLM debug helper.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from types import ModuleType +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +TOOL_PATH = REPO_ROOT / "tools" / "vllm_debug.py" + + +def load_tool() -> ModuleType: + spec = importlib.util.spec_from_file_location("vllm_debug_tool", TOOL_PATH) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_build_serve_command_includes_lora_and_gpu_options() -> None: + tool = load_tool() + + command = tool.build_serve_command( + tool.ServeRequest( + model="HuggingFaceTB/SmolLM3-3B", + host="127.0.0.1", + port=8000, + served_model_name="anonymizer-local", + api_key="test-token", + adapter=Path("/models/adapter"), + adapter_name="anonymizer", + tensor_parallel_size=2, + gpu_memory_utilization=0.8, + max_model_len=4096, + eager=True, + ) + ) + + assert command == [ + sys.executable, + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + "HuggingFaceTB/SmolLM3-3B", + "--host", + "127.0.0.1", + "--port", + "8000", + "--served-model-name", + "anonymizer-local", + "--api-key", + "test-token", + "--enable-lora", + "--lora-modules", + "anonymizer=/models/adapter", + "--tensor-parallel-size", + "2", + "--gpu-memory-utilization", + "0.8", + "--max-model-len", + "4096", + "--enforce-eager", + ] + + +def test_build_serve_command_can_use_a_separate_vllm_interpreter() -> None: + tool = load_tool() + + command = tool.build_serve_command( + tool.ServeRequest( + model="local-model", + python_executable=Path("/opt/vllm/bin/python"), + ) + ) + + assert command[:5] == [ + "/opt/vllm/bin/python", + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + "local-model", + ] + + +def test_render_serve_command_redacts_api_key() -> None: + tool = load_tool() + + rendered = tool.render_serve_command( + tool.build_serve_command(tool.ServeRequest(model="local-model", api_key="test-token")) + ) + + assert "test-token" not in rendered + assert "--api-key ''" in rendered + + +def test_resolve_api_key_reads_a_named_environment_variable(monkeypatch: Any) -> None: + tool = load_tool() + monkeypatch.setenv("LOCAL_VLLM_API_KEY", "test-token") + + assert tool.resolve_api_key(None, "LOCAL_VLLM_API_KEY") == "test-token" + + +def test_discover_cached_models_returns_snapshot_paths(tmp_path: Path) -> None: + tool = load_tool() + snapshot = tmp_path / "models--HuggingFaceTB--SmolLM3-3B" / "snapshots" / "abc123" + snapshot.mkdir(parents=True) + (snapshot / "config.json").write_text("{}", encoding="utf-8") + + models = tool.discover_cached_models(tmp_path) + + assert models == [ + tool.CachedModel( + repository="HuggingFaceTB/SmolLM3-3B", + snapshot_path=snapshot, + ) + ] + + +def test_fetch_models_uses_openai_models_endpoint(monkeypatch: Any) -> None: + tool = load_tool() + calls: list[str] = [] + + class Response: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return {"data": [{"id": "local-model"}]} + + def fake_get(url: str, *, timeout: float) -> Response: + calls.append(url) + assert timeout == 10.0 + return Response() + + monkeypatch.setattr(tool.httpx, "get", fake_get) + + assert tool.fetch_models("http://127.0.0.1:8000/v1", timeout_seconds=10.0) == ["local-model"] + assert calls == ["http://127.0.0.1:8000/v1/models"] + + +def test_call_chat_sends_prompt_and_returns_content(monkeypatch: Any) -> None: + tool = load_tool() + request_body: dict[str, object] = {} + + class Response: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, object]: + return { + "choices": [{"message": {"content": "hello"}}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1}, + } + + def fake_post(url: str, *, json: dict[str, object], timeout: float, headers: dict[str, str]) -> Response: + request_body.update(json) + assert url == "http://127.0.0.1:8000/v1/chat/completions" + assert timeout == 15.0 + assert headers == {} + return Response() + + monkeypatch.setattr(tool.httpx, "post", fake_post) + + result = tool.call_chat( + endpoint="http://127.0.0.1:8000/v1", + model="local-model", + prompt="Say hello", + timeout_seconds=15.0, + api_key=None, + ) + + assert request_body == { + "model": "local-model", + "messages": [{"role": "user", "content": "Say hello"}], + } + assert result.content == "hello" + assert result.usage == {"prompt_tokens": 3, "completion_tokens": 1} diff --git a/tools/vllm_debug.py b/tools/vllm_debug.py new file mode 100644 index 00000000..593b5040 --- /dev/null +++ b/tools/vllm_debug.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Launch and probe local vLLM servers for Anonymizer development. + +Usage: + uv run python tools/vllm_debug.py models --cached + uv run --with vllm python tools/vllm_debug.py serve /path/to/model --dry-run + uv run python tools/vllm_debug.py serve /path/to/model --vllm-python /opt/vllm/bin/python + uv run python tools/vllm_debug.py models --endpoint http://127.0.0.1:8000/v1 + uv run python tools/vllm_debug.py call --model local-model --prompt "Hello" + +The helper does not prefetch models. ``serve`` requires vLLM to be installed in +the Python environment that launches this script. Use a cached snapshot path +from ``models --cached`` to avoid a Hugging Face download; a model ID may cause +vLLM to download it. +""" + +from __future__ import annotations + +import json +import os +import shlex +import subprocess +import sys +from pathlib import Path +from typing import Annotated, Any + +import cyclopts +import httpx +from pydantic import BaseModel + +app = cyclopts.App(help=__doc__) + +DEFAULT_ENDPOINT = "http://127.0.0.1:8000/v1" + + +class ServeRequest(BaseModel): + """Arguments for an OpenAI-compatible vLLM server.""" + + model: str + host: str = "127.0.0.1" + port: int = 8000 + adapter: Path | None = None + adapter_name: str | None = None + tensor_parallel_size: int | None = None + gpu_memory_utilization: float | None = None + max_model_len: int | None = None + eager: bool = False + python_executable: Path | None = None + served_model_name: str | None = None + api_key: str | None = None + + +class CachedModel(BaseModel): + """A Hugging Face cache snapshot usable as a local vLLM model path.""" + + repository: str + snapshot_path: Path + + +class CallResult(BaseModel): + """Normalized OpenAI-compatible chat response.""" + + content: str + usage: dict[str, Any] + + +def build_serve_command(request: ServeRequest) -> list[str]: + """Build the vLLM server command without starting a process.""" + command = [ + str(request.python_executable or sys.executable), + "-m", + "vllm.entrypoints.openai.api_server", + "--model", + request.model, + "--host", + request.host, + "--port", + str(request.port), + ] + if request.served_model_name is not None: + command.extend(["--served-model-name", request.served_model_name]) + if request.api_key is not None: + command.extend(["--api-key", request.api_key]) + _append_adapter(command, request) + _append_gpu_options(command, request) + return command + + +def render_serve_command(command: list[str]) -> str: + """Render a server command without exposing its API key.""" + rendered = command.copy() + if "--api-key" in rendered: + key_index = rendered.index("--api-key") + 1 + if key_index < len(rendered): + rendered[key_index] = "" + return shlex.join(rendered) + + +def _append_adapter(command: list[str], request: ServeRequest) -> None: + if request.adapter is None: + return + adapter_name = request.adapter_name or request.adapter.name + command.extend(["--enable-lora", "--lora-modules", f"{adapter_name}={request.adapter}"]) + + +def _append_gpu_options(command: list[str], request: ServeRequest) -> None: + options = [ + ("--tensor-parallel-size", request.tensor_parallel_size), + ("--gpu-memory-utilization", request.gpu_memory_utilization), + ("--max-model-len", request.max_model_len), + ] + for flag, value in options: + if value is not None: + command.extend([flag, str(value)]) + if request.eager: + command.append("--enforce-eager") + + +def discover_cached_models(cache_root: Path) -> list[CachedModel]: + """Return all snapshot directories in a Hugging Face hub cache.""" + if not cache_root.exists(): + return [] + models: list[CachedModel] = [] + for model_dir in sorted(cache_root.glob("models--*")): + repository = model_dir.name.removeprefix("models--").replace("--", "/") + for snapshot in sorted((model_dir / "snapshots").glob("*")): + if snapshot.is_dir(): + models.append(CachedModel(repository=repository, snapshot_path=snapshot)) + return models + + +def default_cache_root() -> Path: + """Resolve the Hugging Face hub cache without creating it.""" + if hub_cache := os.getenv("HF_HUB_CACHE"): + return Path(hub_cache) + if hf_home := os.getenv("HF_HOME"): + return Path(hf_home) / "hub" + return Path.home() / ".cache" / "huggingface" / "hub" + + +def fetch_models(endpoint: str, *, timeout_seconds: float) -> list[str]: + """Fetch model IDs from an OpenAI-compatible ``/v1/models`` endpoint.""" + response = httpx.get(f"{normalize_endpoint(endpoint)}/models", timeout=timeout_seconds) + response.raise_for_status() + payload = response.json() + return [str(item["id"]) for item in payload.get("data", []) if isinstance(item, dict) and "id" in item] + + +def call_chat( + *, + endpoint: str, + model: str, + prompt: str, + timeout_seconds: float, + api_key: str | None, +) -> CallResult: + """Send a single chat completion request and normalize the response.""" + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + payload = {"model": model, "messages": [{"role": "user", "content": prompt}]} + response = httpx.post( + f"{normalize_endpoint(endpoint)}/chat/completions", + json=payload, + timeout=timeout_seconds, + headers=headers, + ) + response.raise_for_status() + body = response.json() + content = str(body["choices"][0]["message"].get("content", "")) + usage = body.get("usage", {}) + return CallResult(content=content, usage=usage if isinstance(usage, dict) else {}) + + +def normalize_endpoint(endpoint: str) -> str: + """Ensure an endpoint has one ``/v1`` suffix and no trailing slash.""" + stripped = endpoint.rstrip("/") + return stripped if stripped.endswith("/v1") else f"{stripped}/v1" + + +def resolve_api_key(api_key: str | None, api_key_env: str | None) -> str | None: + """Return an explicit API key or one read from a named environment variable.""" + if api_key is not None and api_key_env is not None: + raise ValueError("Use either api_key or api_key_env, not both.") + if api_key_env is None: + return api_key + value = os.getenv(api_key_env) + if value is None: + raise ValueError(f"Environment variable {api_key_env!r} is not set.") + return value + + +def render(value: BaseModel | list[str] | list[CachedModel], *, json_output: bool) -> str: + """Render command results in JSON or compact human-readable text.""" + if json_output: + return json.dumps(_json_value(value), indent=2) + if isinstance(value, list): + return "\n".join(str(item) for item in value) or "No models found." + return value.model_dump_json(indent=2) + + +def _json_value(value: BaseModel | list[str] | list[CachedModel]) -> Any: + if isinstance(value, BaseModel): + return value.model_dump(mode="json") + return [item.model_dump(mode="json") if isinstance(item, BaseModel) else item for item in value] + + +@app.command +def serve( + model: str, + *, + host: str = "127.0.0.1", + port: int = 8000, + adapter: Path | None = None, + adapter_name: str | None = None, + tensor_parallel_size: int | None = None, + gpu_memory_utilization: float | None = None, + max_model_len: int | None = None, + eager: bool = False, + vllm_python: Annotated[Path | None, cyclopts.Parameter("--vllm-python")] = None, + served_model_name: str | None = None, + api_key: str | None = None, + api_key_env: str | None = None, + dry_run: Annotated[bool, cyclopts.Parameter("--dry-run")] = False, +) -> None: + """Launch an OpenAI-compatible vLLM server from the current Python environment.""" + request = ServeRequest( + model=model, + host=host, + port=port, + adapter=adapter, + adapter_name=adapter_name, + tensor_parallel_size=tensor_parallel_size, + gpu_memory_utilization=gpu_memory_utilization, + max_model_len=max_model_len, + eager=eager, + python_executable=vllm_python, + served_model_name=served_model_name, + api_key=resolve_api_key(api_key, api_key_env), + ) + command = build_serve_command(request) + if dry_run: + print(render_serve_command(command)) + return + try: + subprocess.run(command, check=True) + except FileNotFoundError as exc: + raise SystemExit(f"vLLM Python executable not found: {command[0]}") from exc + + +@app.command +def models( + *, + endpoint: str = DEFAULT_ENDPOINT, + cached: Annotated[bool, cyclopts.Parameter("--cached")] = False, + cache_root: Path | None = None, + timeout_seconds: float = 10.0, + json_output: Annotated[bool, cyclopts.Parameter("--json")] = False, +) -> None: + """List served models or cached Hugging Face snapshots.""" + if cached: + print(render(discover_cached_models(cache_root or default_cache_root()), json_output=json_output)) + return + print(render(fetch_models(endpoint, timeout_seconds=timeout_seconds), json_output=json_output)) + + +@app.command +def call( + model: str, + prompt: str, + *, + endpoint: str = DEFAULT_ENDPOINT, + api_key_env: str | None = None, + timeout_seconds: float = 60.0, + json_output: Annotated[bool, cyclopts.Parameter("--json")] = False, +) -> None: + """Send one OpenAI-compatible chat completion request.""" + api_key = resolve_api_key(None, api_key_env) + result = call_chat( + endpoint=endpoint, + model=model, + prompt=prompt, + timeout_seconds=timeout_seconds, + api_key=api_key, + ) + print(render(result, json_output=json_output)) + + +if __name__ == "__main__": + app() diff --git a/uv.lock b/uv.lock index 1cdea812..deede134 100644 --- a/uv.lock +++ b/uv.lock @@ -1,10 +1,15 @@ version = 1 -revision = 2 +revision = 3 requires-python = ">=3.11" resolution-markers = [ - "python_full_version >= '3.14'", - "python_full_version >= '3.12' and python_full_version < '3.14'", - "python_full_version < '3.12'", + "python_full_version >= '3.14' and sys_platform == 'darwin'", + "python_full_version >= '3.14' and sys_platform != 'darwin'", + "python_full_version == '3.13.*' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.13.*' and sys_platform != 'darwin'", + "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "python_full_version < '3.12' and sys_platform == 'darwin'", + "python_full_version < '3.12' and sys_platform != 'darwin'", ] [manifest] @@ -157,6 +162,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, ] +[[package]] +name = "anthropic" +version = "0.121.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "docstring-parser" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0f/ca/3cb2c20ee729736fbd4546d5d8b67e818288529fe70cb7a80dbf80aef70b/anthropic-0.121.0.tar.gz", hash = "sha256:e79d6e08ab3376602fc9a70d4d5ea3540817c76cf7e16658bed790834e1833d6", size = 1013292, upload-time = "2026-08-07T17:11:07.241Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/91/b3d41643f1f639927e8c5fb02c3bd8bffe6f1f29e219b3bd4c61e267b15c/anthropic-0.121.0-py3-none-any.whl", hash = "sha256:6048713fa441e59e1cba8363171cd2a86273b25bd213e9c7ac70a523af88b011", size = 1035493, upload-time = "2026-08-07T17:11:08.508Z" }, +] + [[package]] name = "anyascii" version = "0.3.3" @@ -179,6 +203,29 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" }, ] +[[package]] +name = "apache-tvm-ffi" +version = "0.1.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/60/1e787a0b5ebf318483235be2a689ee367173983067e441b8379564f667c0/apache_tvm_ffi-0.1.9.tar.gz", hash = "sha256:d2d402587e8906de0a07f4746aa78f3d452c7efe3625d4bb39ac2ad693bce530", size = 2513731, upload-time = "2026-02-27T19:28:06.602Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/b1/9f2cfd6d49b03c5d4ec5c12548d911e2e01265be783f343103b4df716765/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c0449fc3802987c3652bea266ffda2934a6f69c80bba791a3f55b91040656a18", size = 2231154, upload-time = "2026-02-27T19:27:15.691Z" }, + { url = "https://files.pythonhosted.org/packages/55/43/63faedea83494e99122466a993bcdccd31cf93c7e8a0d56731120e82e2b9/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f16d73a82a9e68a439b7d233d48b1b929be17fe92df4bbf1ee2274e573144a3", size = 2323130, upload-time = "2026-02-27T19:27:17.259Z" }, + { url = "https://files.pythonhosted.org/packages/27/96/d735bc4c528efaf0a8a954076963c727aad2dde8577641aa9025ec4f2d52/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01ebb1308b2666c206aa9a4015eb48f03a5d98ea2e9cfb002bd5e2ca0b9c7ef3", size = 2159854, upload-time = "2026-02-27T19:27:18.789Z" }, + { url = "https://files.pythonhosted.org/packages/e4/3b/6cfc82a3ab5d9e501bbcee5df36eebe09da1c384461d7a55e2a17776d117/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21365abd2a2a1a6d3b4e6e4f048309651125becfa795440c3607f3cc27d30ac7", size = 2307140, upload-time = "2026-02-27T19:27:20.222Z" }, + { url = "https://files.pythonhosted.org/packages/a7/c0/6d3d54f50012255b41bc3e24944c086f63c4707c8686c7c6780e9283eb96/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d503029e66c43b1a1cb1a42a1e9bb428c8a28dcbdec31c28e705472ca648a3a", size = 2203712, upload-time = "2026-02-27T19:27:25.867Z" }, + { url = "https://files.pythonhosted.org/packages/c6/dd/2bab4c6cd86257dbf99e93452a1af833113f8dc3e25a25579f6e4e4c8a94/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28241371934ea8af10d5067087ba1229ebddded7b2c02d33a258ec2a96df8c46", size = 2299704, upload-time = "2026-02-27T19:27:27.477Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4a/b469bcb2e1014cb84d336d2a59f42958a058251c577a4c2680cacad346e2/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87cacce81df55685fc6a76e1e3c5db1200e85e87bf5974b692c59d131b7bc622", size = 2130865, upload-time = "2026-02-27T19:27:29.092Z" }, + { url = "https://files.pythonhosted.org/packages/70/ef/5402da5d37f5270fd88ea0348acca78dba9be8bdbf6c2bcae0935eb03ef1/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f45eb43499acac45ff6c93564f0ff2d3ca27b69656d540fd56ce59d51c0b4c65", size = 2278991, upload-time = "2026-02-27T19:27:30.729Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a7/1e0643949e683fb3cfababd87058c0cfef122d1a3bb6ce703f719051b842/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d1f4d2b7ec7b1213632e9a104e9330bfc3dec48decffa62114c33aa188c9f43a", size = 2215954, upload-time = "2026-02-27T19:27:35.872Z" }, + { url = "https://files.pythonhosted.org/packages/d6/06/5016191ab61d2db4c3a7d754a3c1184e0836f575a7d08491669738c5e4b9/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4f01d16ba53fe118e363f7257253f07003797e4abe6fc9567f23b6a930dbff2", size = 2307291, upload-time = "2026-02-27T19:27:37.527Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f5/40bf0667330938efbfc0a51743cc53c79e41b4ece1a8abad3076192c9674/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c0581dd6bfbce7b017ef85cfda08bbe38891cc4b3afbcfaa8bc2d383728e426", size = 2143850, upload-time = "2026-02-27T19:27:40.437Z" }, + { url = "https://files.pythonhosted.org/packages/72/4a/421cbd4ed32e8bad3b88af3e8fa145c1f6f493bdd05be15b6f2d9b3cb7d6/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dfa14be2a49347791ef21222a8225ce7f99bfec17104a676cb4f1bf3a107088", size = 2289038, upload-time = "2026-02-27T19:27:41.972Z" }, +] + [[package]] name = "appnope" version = "0.1.4" @@ -256,6 +303,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/d0/7b958df957e4827837b590944008f0b28078f552b451f7407b4b3d54f574/asciichartpy-1.5.25-py2.py3-none-any.whl", hash = "sha256:33c417a3c8ef7d0a11b98eb9ea6dd9b2c1b17559e539b207a17d26d4302d0258", size = 7228, upload-time = "2020-08-17T02:07:16.386Z" }, ] +[[package]] +name = "astor" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/21/75b771132fee241dfe601d39ade629548a9626d1d39f333fde31bc46febe/astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e", size = 35090, upload-time = "2019-12-10T01:50:35.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/88/97eef84f48fa04fbd6750e62dcceafba6c63c81b7ac1420856c8dcc0a3f9/astor-0.8.1-py2.py3-none-any.whl", hash = "sha256:070a54e890cefb5b3739d19f30f5a5ec840ffc9c50ffa7d23cc9fc1a38ebbfc5", size = 27488, upload-time = "2019-12-10T01:50:33.628Z" }, +] + [[package]] name = "asttokens" version = "3.0.1" @@ -319,6 +375,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/39/47f9197bdd44df24d67ac8893641e16f386c984a0619ef2ee4c51fbbc019/beautifulsoup4-4.14.3-py3-none-any.whl", hash = "sha256:0918bfe44902e6ad8d57732ba310582e98da931428d231a5ecb9e7c703a735bb", size = 107721, upload-time = "2025-11-30T15:08:24.087Z" }, ] +[[package]] +name = "blake3" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/26/6a/4cc5a9dd40fd8a6d283fd3761e5f59c490109571ef8e3c73245417e5a305/blake3-1.0.9.tar.gz", hash = "sha256:5fa374fa5070ca084368776c19b420157eb0f2d3f091343d6bc59189929d62e2", size = 116872, upload-time = "2026-06-22T18:02:25.366Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/d6/d5462ec19a7f3d084fe327e08618fa107799ee708df04b3a2d620bd62816/blake3-1.0.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ee96daaa850700fd342a811fa10a8780fd2e8464a71b83a1779c7b6becd3dd5", size = 377621, upload-time = "2026-06-22T18:00:18.389Z" }, + { url = "https://files.pythonhosted.org/packages/92/98/dbc433f2a45be1b2344a6035d4212dfb6e6eb45046ad15103ead9c82d491/blake3-1.0.9-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:09deb024cd75cb200e7f647cd038800e6edc8f190c8188e0c69ec1c2b920e125", size = 377495, upload-time = "2026-06-22T18:00:20.067Z" }, + { url = "https://files.pythonhosted.org/packages/e0/3d/c7a699fb60d8ed31f3f28e6aec7658d29e45ec89e7054906b3040ce3ee65/blake3-1.0.9-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c99afb0459c82dd13e456b6b68d45c4768b539ca998dacd3ed726f1e75e91dc", size = 451158, upload-time = "2026-06-22T18:00:21.459Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a1/0b1b0dbf2dd772483e372237bb65385602b019e24b67424b1fc9e5447837/blake3-1.0.9-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:28528d1f29e6f3d45faf3482e1197e5e175730eef38bdc74e56ee11b68e0ad0d", size = 491988, upload-time = "2026-06-22T18:00:22.984Z" }, + { url = "https://files.pythonhosted.org/packages/ee/d1/ed319477f6d263a4f6b7e9aa465b06be5235a854923edbc9ea09508b6638/blake3-1.0.9-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65c0c20014df687694af5ccf0cec3bdb194511da8ebd50c30b0fd55c83fa4fd5", size = 386848, upload-time = "2026-06-22T18:00:24.319Z" }, + { url = "https://files.pythonhosted.org/packages/80/3e/a4cfb269f3e0955598b415a7843c358c4f79e826e3c9118dc9fb1f101ee6/blake3-1.0.9-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:964b642631a3c8fe117b3439c8ae64a9a0981af9444e409656d1f1e464bfa125", size = 387842, upload-time = "2026-06-22T18:00:25.589Z" }, + { url = "https://files.pythonhosted.org/packages/59/0e/d4ee3d89eece42f86eb46663aa42702000516b7ffbc53f60b918efe95b57/blake3-1.0.9-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:2fd000708662b04be211a22c1095b65fe399d7276e9f3bb2fd1ef8aacc545791", size = 384317, upload-time = "2026-06-22T18:00:26.891Z" }, + { url = "https://files.pythonhosted.org/packages/3a/aa/317106349d10de3b51332ad1e761f4864ebe887854396b75975304dcfbd1/blake3-1.0.9-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:82ecade6ac425fdfc39a4371d6d9232fd6e5c28748fd8d3489016ead17407014", size = 553005, upload-time = "2026-06-22T18:00:28.246Z" }, + { url = "https://files.pythonhosted.org/packages/39/cc/7fbce61a0b24bda1aac99da674bd74ac2b687b61db071c888ffdb30cb47a/blake3-1.0.9-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b4102ba86b86c992a931b4a88c58a632d6097461e14a1e63ebd2ecb98ff0898f", size = 595086, upload-time = "2026-06-22T18:00:29.96Z" }, + { url = "https://files.pythonhosted.org/packages/f2/78/66580635d744c826671fd219938caffb16281a26f62c4f856695d4233677/blake3-1.0.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42fa57bf462285ef16400601b0fd32214c248ba92505bbb94b1221ab9af5a092", size = 373795, upload-time = "2026-06-22T18:00:36.887Z" }, + { url = "https://files.pythonhosted.org/packages/b1/79/b5b17d3004bb81a5732c0b176c812703d200ed8c652b3b7713b9633bbe10/blake3-1.0.9-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b25ccde5a64be070f20e5c7a81da70292db40b164b6c77588cbd6230856badbb", size = 374183, upload-time = "2026-06-22T18:00:38.205Z" }, + { url = "https://files.pythonhosted.org/packages/3c/63/0d209c44b2041bbe130ced12a23c92dd995fbfe5bce7ee77fffea16f5cb0/blake3-1.0.9-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2a800b87433955f37691b5f361ad29c7dd3ee089c9cd109adc5aea8e24bc4c1f", size = 446783, upload-time = "2026-06-22T18:00:39.493Z" }, + { url = "https://files.pythonhosted.org/packages/c5/51/efd1f9b8a9d3e9a0e235f3ced99a738529a1019fe78b3988e29d9c2fbba6/blake3-1.0.9-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6879739e7904b9c42afbedbcc2e8c36cebe140fb3fc3f5c492993579cf5cd516", size = 487369, upload-time = "2026-06-22T18:00:40.875Z" }, + { url = "https://files.pythonhosted.org/packages/8d/3f/a8dcaea9e0b26e419a540ca0cd6203c9fbb505e85b02b03c5a59bf9e6a45/blake3-1.0.9-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6edeb3d49a24c307995899b70dd47aa901d0e9ad51d2f8a79aba4f074f32d8c5", size = 383845, upload-time = "2026-06-22T18:00:42.251Z" }, + { url = "https://files.pythonhosted.org/packages/f6/10/e9907f5b86410d5071982aaf05d149ca4d4fd8acab7e77eebbc9a333c7b4/blake3-1.0.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bcd56a7a972c4185070f7042ccc20166927eec3c0f98b8405f375d007b604a0b", size = 383851, upload-time = "2026-06-22T18:00:43.715Z" }, + { url = "https://files.pythonhosted.org/packages/34/cf/c7863a185550706a9624f6aa7b6d46470aaed0bb46a827c5cda2a7d03151/blake3-1.0.9-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:a288664d08dee154cc496e06e62517fc9e655ecec12b0d7db538d244ac79edf1", size = 380067, upload-time = "2026-06-22T18:00:45.249Z" }, + { url = "https://files.pythonhosted.org/packages/54/0a/e7af679c719368b400c9ba9c3460072aac2ba077ddbd4bc806fef28cda03/blake3-1.0.9-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:91db52a809b68b5bebe7c413ddcd230e1f759398e7fa7a873104595a4fa648b6", size = 549471, upload-time = "2026-06-22T18:00:46.793Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3c/37c1dd3539b7bd9b6d2eef019802aacdb4a3d48ab484b140603bbf9c5b5a/blake3-1.0.9-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:cfaa671b07eb73883162ca940442193868358b0b904cfa266e4b74131ce966da", size = 591396, upload-time = "2026-06-22T18:00:48.122Z" }, + { url = "https://files.pythonhosted.org/packages/5e/aa/0a6967ff9a6ae182419a681aed54f7338b34a1f71372e90f787a2afa42e6/blake3-1.0.9-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:766f1555cbe614f14f399c2fbec0983568d20edb36837ba04040807eb9e1a609", size = 373616, upload-time = "2026-06-22T18:00:54.701Z" }, + { url = "https://files.pythonhosted.org/packages/1c/51/5d4e198bf3ae902c6697ad6ec77d7210736ad8f680980e8b648dcfcd09a0/blake3-1.0.9-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:128a62136c9a39c7cb9fdaa5fb38471f2418853da7f5a89f31495735d0ba6f2c", size = 374149, upload-time = "2026-06-22T18:00:56.015Z" }, + { url = "https://files.pythonhosted.org/packages/7e/62/d3c7c364925b3f10828e5137376f3947f112c32188e899b42f09c2fde98a/blake3-1.0.9-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d1ea0bf17b184b03444007646d902207d2b4d4f3e91a0cac3836552d83db74b9", size = 446151, upload-time = "2026-06-22T18:00:57.378Z" }, + { url = "https://files.pythonhosted.org/packages/b1/01/55b89389c5036c9d24b1d762d6265e91552e10b76a3c99fece3c4a7a4783/blake3-1.0.9-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:73a48f7e9f0e047f51a445d9b0361ab1907bdc72b6857815a84dacd2e59556f8", size = 487256, upload-time = "2026-06-22T18:00:58.763Z" }, + { url = "https://files.pythonhosted.org/packages/b2/7a/a21b52253292ad3e4df63ea4a01ce11d3ee8f4a8a8d80eaf0c7ce92a62bd/blake3-1.0.9-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b27550ada40f839aca64c66127940e4318bb6ef3e291890ef913017f6f637448", size = 383977, upload-time = "2026-06-22T18:01:00.192Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f0/fe7188201a29ee9b042616c786a98afd864d537ca96198e64c3fe4ff13a9/blake3-1.0.9-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66c84dbc2a31eda88b55bbf5c5b711037bf0698eba0fd1faf06bdaf313c39048", size = 383615, upload-time = "2026-06-22T18:01:01.557Z" }, + { url = "https://files.pythonhosted.org/packages/22/08/f6a213b950e30fe9ef7d7fc061ec388e66ed62643570226882e6f7136ea3/blake3-1.0.9-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:dab59b324aa65c09e937d6c43de5de85ec9581627f4e79dcc9806d85b54a1c34", size = 380288, upload-time = "2026-06-22T18:01:03.025Z" }, + { url = "https://files.pythonhosted.org/packages/f1/fb/b171e47c1b835483bcf1545ebc289458165f8dc0f5c7f74a9176d7e9af03/blake3-1.0.9-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:eca281fedcbe5c56655bd5a4176e6036eddbbe57df96114a03838fce08b1e0ca", size = 549122, upload-time = "2026-06-22T18:01:04.486Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d8/7bf71c2c85a0951e406971f151435e0751716907e3924c6c48a2d6dae0db/blake3-1.0.9-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:3cbe7f190164896dc3908e920716ee66bc31d40f1a0fb603ed59ac53290fb9cf", size = 591183, upload-time = "2026-06-22T18:01:06.259Z" }, + { url = "https://files.pythonhosted.org/packages/20/f1/d03950a86d105a6332a8c422cb87658a7d247e214f1ea8f29ed09ff04e00/blake3-1.0.9-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95fc3545f80901b0dcd0508d16bc40f15ae39556709fa6cf86675f742d4f3c9c", size = 375147, upload-time = "2026-06-22T18:01:13.198Z" }, + { url = "https://files.pythonhosted.org/packages/10/75/711b1842e0a90aaad6a1c9a9022e90aa16206ac1f224516118bc24482532/blake3-1.0.9-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1bd981dc318c05375c3160a99df493b7cc4c83fffa1a34d14b18a071b47b262b", size = 373711, upload-time = "2026-06-22T18:01:14.606Z" }, + { url = "https://files.pythonhosted.org/packages/9e/a0/f512799d1d0c0b4718fa6f0e99ccbe108e98bac7bf82c200803a62b57876/blake3-1.0.9-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:689a7e4069de681d9c5d9445b8b6473ee880ad04d7960a6789c60bd788980250", size = 446993, upload-time = "2026-06-22T18:01:15.924Z" }, + { url = "https://files.pythonhosted.org/packages/60/fb/6636ae8a46fc3352694188f5a5a325567782bc88fd1823b0b67be2c92184/blake3-1.0.9-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8adb0b0032e53919ee95b3d4f911448d3268316c28cd7df232ff2a1e7c9a4ba4", size = 488478, upload-time = "2026-06-22T18:01:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/1e/c5/a2b3c086f7e37c9db6017dc2890a76ad2a729e4a554896e855e511811e6b/blake3-1.0.9-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32bd4521ec2d477627ad93eb70f9ac4d01e12d1489024159bcaeff79466332f6", size = 384900, upload-time = "2026-06-22T18:01:18.802Z" }, + { url = "https://files.pythonhosted.org/packages/e1/b8/1298806dd6c464a6f807df24c9640ad3bf27ee54ff4de82b2b5a823a8aba/blake3-1.0.9-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f65d77eb05331495485048f6804f53885b192b998acb7e6fe1487d941bf08435", size = 384333, upload-time = "2026-06-22T18:01:20.35Z" }, + { url = "https://files.pythonhosted.org/packages/3c/cc/0c29d9404155adfd6db716e9765d36ea6cbed287060759f5d764f0d9d99e/blake3-1.0.9-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:ca7dfe8fb197ff8a3f5c915424183ccd52a99e8afb12680f51b2e1f4c9c6c97f", size = 381142, upload-time = "2026-06-22T18:01:21.744Z" }, + { url = "https://files.pythonhosted.org/packages/d6/91/9af20d563f0ced71e08a60fc0ee534146da4e265710ed6792d5d799f4c0f/blake3-1.0.9-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:f5c9d57f0dcb92243b6ae575c3065793edc9df9008d0ebd98d8245cdeb7c3f84", size = 550587, upload-time = "2026-06-22T18:01:23.381Z" }, + { url = "https://files.pythonhosted.org/packages/d0/fa/06f46fc0aa486b799d776f9a80ed0b3605e2be1570cf48007860948aa5d9/blake3-1.0.9-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:172d44245a19dfec08ab771c1b7a506b97783163cdc65f559fe020007e403c99", size = 591888, upload-time = "2026-06-22T18:01:24.805Z" }, + { url = "https://files.pythonhosted.org/packages/9d/da/e25fa75d5bfea4527fc21024dde86a9376db798e469a084741968299f215/blake3-1.0.9-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09a69fcedf06785bb81d4d3d39f95ee65dbaf2cb246e174cfc9ff64d027f7551", size = 374203, upload-time = "2026-06-22T18:01:31.998Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4d/0224916202b773dfdf08dcbe4ed1ad1018d4ddcd4df7a7e2978d28f89b74/blake3-1.0.9-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5d5bf0f68cd77108a942c95db98e960d9c3d5643b95172f783822ce22667759", size = 373713, upload-time = "2026-06-22T18:01:33.387Z" }, + { url = "https://files.pythonhosted.org/packages/b8/e5/4ba968831b7afaec431c588c826cef76a96d6d6976188ed07d932072e673/blake3-1.0.9-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9767f16199b99aa022b61ff825ac4dbd39864bf637ae712605a2ce1f8b6a55e0", size = 446574, upload-time = "2026-06-22T18:01:34.687Z" }, + { url = "https://files.pythonhosted.org/packages/ac/f5/08a9099c7177f282d2563abe4f7cc626c636642f7979cf58f2ab7ded2096/blake3-1.0.9-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4865a8cfb2b3d7c0baf5267f2fa6816a3384e836cd1bd0caf359f406cb1e8fba", size = 487232, upload-time = "2026-06-22T18:01:36.131Z" }, + { url = "https://files.pythonhosted.org/packages/76/16/9392bf1ebc81b5b09ce58b94613fa2d37308e825ff2dc7b54d00ee622c77/blake3-1.0.9-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:42609e4adc4b2d7423137f2cb35135bca598b925c5af09d2bc0a2c368b25aeb1", size = 384751, upload-time = "2026-06-22T18:01:37.512Z" }, + { url = "https://files.pythonhosted.org/packages/84/fc/b6e9aef02ca14ef62fa47783b9eeeb5b2d3f73fdf698d8bb94c36f5dd69f/blake3-1.0.9-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7f648fa425138452d1e585ac625c7aefddb946d9765906c4c12d564a1523cd8", size = 384546, upload-time = "2026-06-22T18:01:38.868Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cb/452e92dba9402b36a953aa8b9b06253445ccce43dcd0bcf521c5e3c3e15d/blake3-1.0.9-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:9cef6d4d07a7de0c44f5ba17f6383d55276d9efc8d601f75113538fcaa35008b", size = 380596, upload-time = "2026-06-22T18:01:40.412Z" }, + { url = "https://files.pythonhosted.org/packages/b2/01/7a84a7e10c5d14e6ed8a4403bd7f64c1e01f8ebabea0d6fe5f093b894cbd/blake3-1.0.9-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:28404301de485e9546365d01b30f65eaa835520c4211d6ef61242975b6722b60", size = 550032, upload-time = "2026-06-22T18:01:41.955Z" }, + { url = "https://files.pythonhosted.org/packages/58/7d/7aea0222f59cf84044ec52e2bfdaa0e3c355d221292b0ea1b722cf1edd6c/blake3-1.0.9-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:8a99f896e7718050ed033a888245098aab3d6a5338f91cc9450c563b53f90ad5", size = 592244, upload-time = "2026-06-22T18:01:43.426Z" }, +] + [[package]] name = "bleach" version = "6.3.0" @@ -336,6 +448,43 @@ css = [ { name = "tinycss2" }, ] +[[package]] +name = "cachetools" +version = "7.1.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/70/d2/47e8bc06fe2a06d3f5bdf20f1126ab66c4e99dc48d940e7ba873f7ac7131/cachetools-7.1.7.tar.gz", hash = "sha256:a3e2a00b14d8f8a6b70c1dae7b4685e7ad3bc965c5b42124a2d6ce895da6cf50", size = 40680, upload-time = "2026-08-01T21:20:40.434Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/d8/767faeda872075724b95dd675466a645f1b92aadcdcf2d1429dcfd76c176/cachetools-7.1.7-py3-none-any.whl", hash = "sha256:ef98ef375ad188819ef2f9b3645e3987f4b8c5b7550e436ad998c2de78296df0", size = 16830, upload-time = "2026-08-01T21:20:38.977Z" }, +] + +[[package]] +name = "cbor2" +version = "6.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/14/b02446bacfe44351b1689c04937ade007588f44570431880a6937e525e6c/cbor2-6.1.4.tar.gz", hash = "sha256:01ecc79a28f33d17331943ce508fc1e21f4b06553c73f874f4c77120d72b2ef9", size = 90840, upload-time = "2026-08-01T20:41:39.797Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/96/d8e1ed3e79ea20a3423a96b5c89ce794fa02cb428e4429e601f8ebcbac7c/cbor2-6.1.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:e1fe2d62c50df290576280b18247ec63486f78be73e285bae269c2456c6ddff0", size = 457343, upload-time = "2026-08-01T20:40:38.868Z" }, + { url = "https://files.pythonhosted.org/packages/d5/0c/5796c2ed2dcd0696fc4abedf0ea0dfd5361b3f022a311481f977fa51b2b8/cbor2-6.1.4-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c204a75f91f8cd9ed0881f6b88ec395c59aeac9fcf4d08155e7f899db2a1c46e", size = 464314, upload-time = "2026-08-01T20:40:40.63Z" }, + { url = "https://files.pythonhosted.org/packages/b1/88/de524c6c2c91b740e5df6e6955a113fb616e979b26fd2e6a0693082d36e0/cbor2-6.1.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:28fa5db05a7eae8fd80709959988d8a7f12838c6d4e5c58ec951414058641195", size = 523053, upload-time = "2026-08-01T20:40:42.602Z" }, + { url = "https://files.pythonhosted.org/packages/84/07/cb5fd92834633508d680a5b5695aeaf99d33ca0bdc5b844550d538f335b0/cbor2-6.1.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:316e217a496640418d3137483279d0e70053b000cdd4b52a4dbf20ea478bc40a", size = 532177, upload-time = "2026-08-01T20:40:44.058Z" }, + { url = "https://files.pythonhosted.org/packages/96/ac/f58b3bafce7c86ada2ad8eaf189453136d2cf5bae526ea0540e1b9bc9d06/cbor2-6.1.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d9ada5a6ccfbb8ea7a3aa2aeb028421b52d8e0cd9323f0a2aeaa9c09d25fbce2", size = 449851, upload-time = "2026-08-01T20:40:51.725Z" }, + { url = "https://files.pythonhosted.org/packages/f0/a5/10c6c126d59b07f2bd005094dd12a20afa46146f7e2673ed6f61a57641a7/cbor2-6.1.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:310f3dfb296ba48fe9b63c5cf26e691e3548a1eae6901d2f0c18e941d151f220", size = 461193, upload-time = "2026-08-01T20:40:53.446Z" }, + { url = "https://files.pythonhosted.org/packages/15/e4/4445e6237088d1cca3b8536daeb90d6b4e23776de5609c9fa46773874757/cbor2-6.1.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e6c76004d674ad1c620660cb0bc5a8a0b72a5d8c7b70926d8e09e6d7e87332f", size = 516937, upload-time = "2026-08-01T20:40:54.952Z" }, + { url = "https://files.pythonhosted.org/packages/8c/87/9c0959510f7a402e5995c81ccfd82cb9f314140dc0cce88c12836e5b93f1/cbor2-6.1.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:32a4663425fbca4a4a7aa918eb5789d844c406439e58424cf34511f79f559242", size = 529229, upload-time = "2026-08-01T20:40:56.365Z" }, + { url = "https://files.pythonhosted.org/packages/35/3d/93eed770864540c5c9ea0841008208e9db686b7335f42520705b7d6dc6b2/cbor2-6.1.4-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:4bd29f21529e279d50fc14f1a811f7b05b4d8e66a7969163cce98983b6817245", size = 449762, upload-time = "2026-08-01T20:41:04.094Z" }, + { url = "https://files.pythonhosted.org/packages/e3/21/69e4d37f00319b3d37322355aedc83154b4d8b75dc9e9789c06e1fbd8a92/cbor2-6.1.4-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:36ae16d64b1f7b620c1af748e7b6947e20069ef80eee56871c5fbb84cc635905", size = 460420, upload-time = "2026-08-01T20:41:05.891Z" }, + { url = "https://files.pythonhosted.org/packages/be/26/2cfdd5ee826205a88a826bb38b7a572c676ec3efa29574be5cdbd04b4859/cbor2-6.1.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:69978901302ecbc8cda57b520487c5c5240ed217de783eb7728fceb258311d76", size = 516490, upload-time = "2026-08-01T20:41:07.52Z" }, + { url = "https://files.pythonhosted.org/packages/82/86/d687cd1c2c9f9a986e8552ad1fdbd22411cc86389b5705dba6ec6f7e3226/cbor2-6.1.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad4efa23fee6447e56a269191044e06eb39e809458bcd674e164fe9445feafd0", size = 528810, upload-time = "2026-08-01T20:41:09.144Z" }, + { url = "https://files.pythonhosted.org/packages/46/f9/b9f12a5e24d5ae355e4c0f6d37330a2bbedad3331247a223a51c4cd39d5e/cbor2-6.1.4-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:0859a0837e6e2d4fe5f5b849f6475797e4db545da98c19db4b1d3487bd47aa22", size = 452191, upload-time = "2026-08-01T20:41:16.705Z" }, + { url = "https://files.pythonhosted.org/packages/67/22/8224b01f95a6fe07b1a64082aea34d9f49068392b3de93f5f3a10c73c62e/cbor2-6.1.4-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:c0f5f2d6d3b58e44146860c049f3c082207a4005588b8926d51bf937ab66773c", size = 462383, upload-time = "2026-08-01T20:41:18.17Z" }, + { url = "https://files.pythonhosted.org/packages/92/52/437e4aa4f5df1fb41020d64b3d99a8239f0f99a3a75eb6ffa5cb66004b7f/cbor2-6.1.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:239db0f92d537fd29eaec4e40195fc3b2b48bc34a5887059658162489a9eb6ae", size = 518700, upload-time = "2026-08-01T20:41:19.592Z" }, + { url = "https://files.pythonhosted.org/packages/7d/45/2f5ea5bfe0fd800b3739c7df8679bdffa9f7def6b2f2fee064ada1c63e85/cbor2-6.1.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3f4a434c36bb0d33aeb48ddae8e8b673ca7e1f14545ee7cf4a4c7c39380ea9a2", size = 531243, upload-time = "2026-08-01T20:41:21.21Z" }, + { url = "https://files.pythonhosted.org/packages/d5/1a/a8624023b84b41c43a150a89517c104aed0e467bd258866f13be4c3ac0c6/cbor2-6.1.4-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:8f1019494b0ec81a3df3ebb01b6acb446d5b946fe35845b1726379abd66a71da", size = 445301, upload-time = "2026-08-01T20:41:28.35Z" }, + { url = "https://files.pythonhosted.org/packages/60/39/07dd0ea957c1f48673d3947f97ee36826efd4a824053dd0ec4df2f0c89d6/cbor2-6.1.4-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:179a794bf4be1d46ff190695929f65f0b42019c156919846ae539d2a7ec42e54", size = 459816, upload-time = "2026-08-01T20:41:29.839Z" }, + { url = "https://files.pythonhosted.org/packages/23/8e/2015175132a27c1daed434f671ac6d9c1311461995df47f201307700e0da/cbor2-6.1.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9b904b8d0f4ddac9259197d21d121fae4cb8b555700d65bc12c5d46a2e6c2025", size = 511565, upload-time = "2026-08-01T20:41:31.939Z" }, + { url = "https://files.pythonhosted.org/packages/82/66/420991095d9473614b205d4c4e40b5d3b9f1ee4410eb3c48c1e902947837/cbor2-6.1.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:71fcf4f237d68bf4445bf45070f36f82b333f2e6a62612aa2c256683b51378a9", size = 527709, upload-time = "2026-08-01T20:41:33.413Z" }, +] + [[package]] name = "certifi" version = "2026.1.4" @@ -508,14 +657,23 @@ wheels = [ [[package]] name = "click" -version = "8.3.0" +version = "8.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/46/61/de6cd827efad202d7057d93e0fed9294b96952e188f7384832791c7b2254/click-8.3.0.tar.gz", hash = "sha256:e7b8232224eba16f4ebe410c25ced9f7875cb5f3263ffc93cc3e8da705e229c4", size = 276943, upload-time = "2025-09-18T17:32:23.696Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/d4/81420972a676e8ffea40450d8c8c92943e7218a78fe9b64359836cc9876b/click-8.4.2.tar.gz", hash = "sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6", size = 338000, upload-time = "2026-06-24T17:45:15.148Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/e2/79c688af8b210d232694e31e59da9f6ec747bae31c3f5946e4e9b98860d5/click-8.4.2-py3-none-any.whl", hash = "sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76", size = 119243, upload-time = "2026-06-24T17:45:13.73Z" }, +] + +[[package]] +name = "cloudpickle" +version = "3.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/27/fb/576f067976d320f5f0114a8d9fa1215425441bb35627b1993e5afd8111e5/cloudpickle-3.1.2.tar.gz", hash = "sha256:7fda9eb655c9c230dab534f1983763de5835249750e85fbcef43aaa30a9a2414", size = 22330, upload-time = "2025-11-03T09:25:26.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/db/d3/9dcc0f5797f070ec8edf30fbadfb200e71d9db6b84d211e3b2085a7589a0/click-8.3.0-py3-none-any.whl", hash = "sha256:9b9f285302c6e3064f4330c05f05b81945b2a39544279343e6e7c5f27a9baddc", size = 107295, upload-time = "2025-09-18T17:32:22.42Z" }, + { url = "https://files.pythonhosted.org/packages/88/39/799be3f2f0f38cc727ee3b4f1445fe6d5e4133064ec2e4115069418a5bb6/cloudpickle-3.1.2-py3-none-any.whl", hash = "sha256:9acb47f6afd73f60dc1df93bb801b472f05ff42fa6c84167d25cb206be1fbf4a", size = 22228, upload-time = "2025-11-03T09:25:25.534Z" }, ] [[package]] @@ -536,6 +694,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/97/891a0971e1e4a8c5d2b20bbe0e524dc04548d2307fee33cdeba148fd4fc7/comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417", size = 7294, upload-time = "2025-07-25T14:02:02.896Z" }, ] +[[package]] +name = "compressed-tensors" +version = "0.15.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "loguru" }, + { name = "pydantic" }, + { name = "torch" }, + { name = "transformers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/41/1b/c3c4a98ec5f2727656336f07a0c35862195c310d8eb0b2fa5b4be6848680/compressed_tensors-0.15.0.1.tar.gz", hash = "sha256:a8e93054e8a5ec49c980b09ed36c4c1249b4a8ee167920a8e461c4da26e78d99", size = 229412, upload-time = "2026-04-10T14:23:54.708Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/52/93833dc1610e017ac5b7dcd59b8304d8ef67d1114c2d124e728a2cbbea12/compressed_tensors-0.15.0.1-py3-none-any.whl", hash = "sha256:e1b1f322e82e475715e242bad46925a304ea8e5c98b5055a15b8eb22fb6bfea9", size = 194260, upload-time = "2026-04-10T14:23:53.098Z" }, +] + [[package]] name = "coverage" version = "7.13.4" @@ -696,6 +869,132 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/50/a9caea39ad19c431c1a3f8a31114df65b260cdfe67786b6c7e7c040c4c44/cryptography-49.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6", size = 3783731, upload-time = "2026-06-12T20:02:43.319Z" }, ] +[[package]] +name = "cuda-bindings" +version = "13.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/6b/457ca12dad3ee9bfcc9a545cfd6b64b359ba49de40f776f6e028e678f262/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474", size = 6053539, upload-time = "2026-05-29T23:11:43.19Z" }, + { url = "https://files.pythonhosted.org/packages/95/7a/c5e3c34a409b148f5c0f5a4ea374158f95d488862c1dffedf9aa5c639df9/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:04436a9364059c84b8f9636f359eccda1cf814341f5b670c71d80d2f79dbc708", size = 6674166, upload-time = "2026-05-29T23:11:45.478Z" }, + { url = "https://files.pythonhosted.org/packages/ce/67/5e7dba1ba576dd73da5dee894ca076ca5e959450dfff66d6d510a255d1f7/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7855c4868aabc0cfae28abbe83d56734bdfbd08f08fc234ac1912a12858bf49", size = 6025351, upload-time = "2026-05-29T23:11:49.685Z" }, + { url = "https://files.pythonhosted.org/packages/39/2a/6d2e9047d1fb243dbaa364b01e0297534b9ed7fd27dba1c9f361519cf69b/cuda_bindings-13.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e32d08f71ebcdf00f0f41eab2eb37e8da94c8ed411cc9f7f7a019ce6b34abe3a", size = 6657965, upload-time = "2026-05-29T23:11:52.227Z" }, + { url = "https://files.pythonhosted.org/packages/cc/6e/2394f8163360f8391f8f1b7e72d300a82724edb81a7b7084c799fbd4c91f/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9efb21c1ee64981e184b9e0ba5eb3179e5ba3d4b51665a6cb52b8ef3d01a7cbf", size = 5920504, upload-time = "2026-05-29T23:11:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/34/c2/ef9b6a63f7dc432712a462c816662e662e00d38caa9b861c8c2588195d03/cuda_bindings-13.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2732904099e0a4d4db774a5fc6d91ee95fae065b4d2ecabb4968c5fe2406c9d7", size = 6476660, upload-time = "2026-05-29T23:11:59.188Z" }, + { url = "https://files.pythonhosted.org/packages/b1/81/bff68ce829999c1e4209c761bbf903b1c06ec570416ddb25020864ad5907/cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ab2f74ed65bfef4163ba07a8db16f1085e0729291db12a2423aff84ee8278b8", size = 6013639, upload-time = "2026-05-29T23:12:03.509Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e0/c8a1f0c8f9ffdea4f5fe6dbab89b326cef4d85caf489dad39e209da89416/cuda_bindings-13.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd4c814d311ec08c981f6dded1dbe7d4b371067ee4f6c14cccec4bde9590f80", size = 6534419, upload-time = "2026-05-29T23:12:05.633Z" }, + { url = "https://files.pythonhosted.org/packages/52/b8/83b1f563925b290f2d11a01a77a84013ba56052fe3653a5bef3ccfbb43d6/cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c3c772dfff49681541d59630c90f858e173ac926b9c593a2b7123f2a1043cc76", size = 5809771, upload-time = "2026-05-29T23:12:10.422Z" }, + { url = "https://files.pythonhosted.org/packages/12/20/e79b4bfe98f075195afb6343d41c498f9dbd2d161d7021d4d28bceb83581/cuda_bindings-13.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:36febb7c1079d68a981dbbd8d5a67235b399802b82075c9388624719607e52b9", size = 6358584, upload-time = "2026-05-29T23:12:12.767Z" }, +] + +[[package]] +name = "cuda-core" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/4b/4ac1d0639241da756c634add606f93a7f3a39bef12f70e1fb4b40cc53c21/cuda_core-1.0.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3effd11283bc46fd06348c2fd18a0941ba7718a6f447343858c944c1a93a6dab", size = 4784340, upload-time = "2026-05-12T20:11:23.961Z" }, + { url = "https://files.pythonhosted.org/packages/01/55/bb3e701f4af504e5e39e837135dc80022ec4c84858b2886ad577fe696a77/cuda_core-1.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1934517ff8a9dcd21b3f4a28e15e12643164b7d3ec187a4ee7560e22fd2dfc17", size = 5059041, upload-time = "2026-05-12T20:11:26.045Z" }, + { url = "https://files.pythonhosted.org/packages/0d/a0/1daeae599cadd612689dbbf70d7da1c01883964fc2fbc7386f3c630a68cf/cuda_core-1.0.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6816dc020aee6103d8071bc02d8e4e1d91f2b49596f666896d608d92224d79d1", size = 4789856, upload-time = "2026-05-12T20:11:30.862Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4d/603557ab3cb171cc2a61d3678a39cb4dae3fd21275078bfbd1c0b0b5230b/cuda_core-1.0.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be7b65311bf78964b7905adbf3c0f8f717d432f2854dc45169277729bf60f1e2", size = 5106023, upload-time = "2026-05-12T20:11:33.509Z" }, + { url = "https://files.pythonhosted.org/packages/57/f9/a6676b1fa555fad5748a945f4b530b51b898b4771a1e5d9f3520d3f415ea/cuda_core-1.0.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c427e5025096d96fcd5092fdc85d5d5e4ac3dea007914e90472ed52f27220446", size = 4749800, upload-time = "2026-05-12T20:11:38.012Z" }, + { url = "https://files.pythonhosted.org/packages/9c/9d/4534a9564a812ee95b43db7324f9b25cbffda001bb348bb5b3f90dad50b9/cuda_core-1.0.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b392178202c652368883dbe3773cee14f3e1ed6b8bf45d1a1bcdd37c73604e06", size = 5078597, upload-time = "2026-05-12T20:11:40.836Z" }, + { url = "https://files.pythonhosted.org/packages/c2/45/55b07d643c87f1234b3cbc9d8383c0962b368ba1d6686a1919d6f6001af4/cuda_core-1.0.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9b0f115a68f2f84c0f6d9b7e863a29517f6dfe5f7b7d07d1d9da8904754e9a2", size = 4816184, upload-time = "2026-05-12T20:11:46.114Z" }, + { url = "https://files.pythonhosted.org/packages/51/08/1aeffc9a529a7f94c9cee9bfd3a991743398b5f90aab30f06f2a4bc8205e/cuda_core-1.0.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:af2db9e50e81d73e4f0b72ad279d0a9c789372393938fe75c17236b9ed974d7d", size = 5104496, upload-time = "2026-05-12T20:11:48.518Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ab/db09228d5a8c124a93514726d2e18f31824f66d3a6769ee4e51721dd64cf/cuda_core-1.0.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4410bf1ef15c2ec23dccc302da76893c9354b530dee422e3277b116231bf5fe1", size = 4996094, upload-time = "2026-05-12T20:11:53.575Z" }, + { url = "https://files.pythonhosted.org/packages/29/e7/8ced56d6c6fa32b7385a8dccefd1424e3c1201bfee3385d9710609a43d16/cuda_core-1.0.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0f1324486bb90be6bde28bd0afbaf78a38827948d37f93f90318a01da8a3f8c", size = 5212461, upload-time = "2026-05-12T20:11:56.238Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/b4/d088047afe39827556df21118cac9ffd20cc3f968c99a7681494d1eb333c/cuda_pathfinder-1.6.0-py3-none-any.whl", hash = "sha256:1503af579d8379c24bdd65528379bc57039b0455be9f5f9686cf8e473a1fce51", size = 54591, upload-time = "2026-07-21T15:03:56.224Z" }, +] + +[[package]] +name = "cuda-python" +version = "13.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings" }, + { name = "cuda-core" }, + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/31/7ff3f7768eded7535c621abc2fecb9d181a34ea4cae3afe682feb796f242/cuda_python-13.3.1-py3-none-any.whl", hash = "sha256:280b014139ab447b6dd70a377db1596f310d6e887d9d342e6651b919ec145fb3", size = 8295, upload-time = "2026-05-29T23:28:47.012Z" }, +] + +[[package]] +name = "cuda-tile" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/4d/e07fd65640c26c1f990ee621af11f073672e8d96501663026c7e1978f5b8/cuda_tile-1.5.0-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:81b8a93e757258260bd05dbbe6eec8bb50655ee1a88d5ebbc8412c526d3c0ed4", size = 322684, upload-time = "2026-07-08T01:49:21.318Z" }, + { url = "https://files.pythonhosted.org/packages/eb/a8/981c2eb351f15a4b75b5913e35d2ef16cee32a45c950d0e16f85e626dd05/cuda_tile-1.5.0-cp311-cp311-manylinux2014_x86_64.whl", hash = "sha256:9494170237d34bbbce83f2ada005d2dabb704f1f8c6a0af59088c180bd1bf028", size = 324278, upload-time = "2026-07-08T01:49:18.576Z" }, + { url = "https://files.pythonhosted.org/packages/7c/6d/cc2fb5a25689a501564a2eced4acf654f307e801a2c1506be97c0d100491/cuda_tile-1.5.0-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:87652483baa9c81a9a24e4450f016e4ee78fd205d8422dad8996571bd1f2622e", size = 322641, upload-time = "2026-07-08T01:49:23.388Z" }, + { url = "https://files.pythonhosted.org/packages/1c/f5/b4ba9d0fc71198d939ebf9a090228179995d8411ee9def8f638a0e3ccdc5/cuda_tile-1.5.0-cp312-cp312-manylinux2014_x86_64.whl", hash = "sha256:cef6d30acc37557643ece0de3770fc4c33497c4af40209e424f72fbfcbe6ea5a", size = 324990, upload-time = "2026-07-08T01:49:17.739Z" }, + { url = "https://files.pythonhosted.org/packages/00/46/60aea981ee7cc0b159eb08c42b795e8e95ae8a7fb451e4565cad0f43cca0/cuda_tile-1.5.0-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:cfa4a5ef920d9c1fee702611b25bde449915f12bf929c1b8a3faee0c9b03f750", size = 322643, upload-time = "2026-07-08T01:49:26.107Z" }, + { url = "https://files.pythonhosted.org/packages/26/d5/ae03d2b70ed8d6c21ca809ddc98227ad07988e7fe67e7e41d888c0b13d32/cuda_tile-1.5.0-cp313-cp313-manylinux2014_x86_64.whl", hash = "sha256:e7cb56186b0cc98166b72c7e5a3764c236151aa8d53e0b37a153766b79ea005a", size = 324995, upload-time = "2026-07-08T01:49:19.931Z" }, + { url = "https://files.pythonhosted.org/packages/82/f2/861000a1cc2204be90295c980c488670600e89c638138192c61bf79949f1/cuda_tile-1.5.0-cp314-cp314-manylinux2014_aarch64.whl", hash = "sha256:b88d7a3ea0cb30a962c0ff9cb01e2b2b87c6ef816fd53dd92ead3bcff3449e37", size = 322775, upload-time = "2026-07-08T01:49:25.897Z" }, + { url = "https://files.pythonhosted.org/packages/65/38/165499cbfb7c1ada110592fa9224e20851125b337bbe2e656b5d56c676f0/cuda_tile-1.5.0-cp314-cp314-manylinux2014_x86_64.whl", hash = "sha256:d4054dc12b6d35d69cb07fcc224183b84f3ad1a0e85ec4414cf660144b8467b0", size = 325052, upload-time = "2026-07-08T01:49:20.082Z" }, + { url = "https://files.pythonhosted.org/packages/2c/53/e5947ebd44774183f72c317907d803648d8cd87e8a571bb0a4e89a578309/cuda_tile-1.5.0-cp314-cp314t-manylinux2014_aarch64.whl", hash = "sha256:a4ede5529bd5e13318ec9bbf3f79abaf23b6368c58d247eaf2e5eb5ccae3fa73", size = 324979, upload-time = "2026-07-08T01:49:29.759Z" }, + { url = "https://files.pythonhosted.org/packages/d7/b8/6260d8089287dd8e8f5e456a60391537520b20cd90287eacaac4c71fd140/cuda_tile-1.5.0-cp314-cp314t-manylinux2014_x86_64.whl", hash = "sha256:1088a3ebb5622c24ec32034d0d451bab9c1b85ae67fcdc647464951ec1d1b6ad", size = 326849, upload-time = "2026-07-08T01:49:23.211Z" }, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas" }, +] +cudart = [ + { name = "nvidia-cuda-runtime" }, +] +cufft = [ + { name = "nvidia-cufft" }, +] +cufile = [ + { name = "nvidia-cufile" }, +] +cupti = [ + { name = "nvidia-cuda-cupti" }, +] +curand = [ + { name = "nvidia-curand" }, +] +cusolver = [ + { name = "nvidia-cusolver" }, +] +cusparse = [ + { name = "nvidia-cusparse" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc" }, +] +nvtx = [ + { name = "nvidia-nvtx" }, +] + [[package]] name = "cyclopts" version = "4.10.1" @@ -745,7 +1044,8 @@ dependencies = [ { name = "httpx" }, { name = "idna" }, { name = "jinja2" }, - { name = "numpy" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "pandas" }, { name = "pillow" }, { name = "pyarrow" }, @@ -786,7 +1086,8 @@ dependencies = [ { name = "marko" }, { name = "mcp" }, { name = "networkx" }, - { name = "numpy" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "pandas" }, { name = "pyarrow" }, { name = "pydantic" }, @@ -815,7 +1116,8 @@ dependencies = [ { name = "httpx" }, { name = "huggingface-hub" }, { name = "multiprocess" }, - { name = "numpy" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "packaging" }, { name = "pandas" }, { name = "pyarrow" }, @@ -872,6 +1174,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "depyf" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astor" }, + { name = "dill" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/35/83fb0178212279aa0af031031905804c6de5618435d229f41ed21bb9ad2c/depyf-0.20.0.tar.gz", hash = "sha256:fb7683bd72c44f67b56029df2c47721e9a02ffa4d7b19095f1c54c4ebf797a98", size = 6168761, upload-time = "2025-10-13T12:33:38.589Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/65/4df6936130b56e1429114e663e7c1576cf845f3aef1b2dd200c0a5d19dba/depyf-0.20.0-py3-none-any.whl", hash = "sha256:d31effad4261cebecb58955d832e448ace88f432328f95f82fd99c30fd9308d4", size = 39381, upload-time = "2025-10-13T12:33:33.647Z" }, +] + +[[package]] +name = "detect-installer" +version = "0.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/ce/6897d812825e9d4c53e3c7112726e800cc5231b013b2223bf64f653ff362/detect_installer-0.1.0.tar.gz", hash = "sha256:00ad7ba0a36e3cf7d08a40d3643011746dbc112597c7d475cc91c416710ca4e7", size = 3049, upload-time = "2026-02-23T10:40:22.567Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/34/8cc73273414405086c58852916e4031812a6a30fe04c057e37ad99397b7f/detect_installer-0.1.0-py3-none-any.whl", hash = "sha256:034fb20fd665c36e6ba52b8821525ea07fb4f7f938cac459df889fb33801528a", size = 4539, upload-time = "2026-02-23T10:40:23.807Z" }, +] + [[package]] name = "diff-cover" version = "10.2.0" @@ -896,6 +1220,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252, upload-time = "2024-01-27T23:42:14.239Z" }, ] +[[package]] +name = "diskcache" +version = "5.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -905,6 +1238,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, +] + +[[package]] +name = "dnspython" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/8b/57666417c0f90f08bcafa776861060426765fdb422eb10212086fb811d26/dnspython-2.8.0.tar.gz", hash = "sha256:181d3c6996452cb1189c4046c61599b84a5a86e099562ffde77d26984ff26d0f", size = 368251, upload-time = "2025-09-07T18:58:00.022Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, +] + [[package]] name = "docstring-parser" version = "0.17.0" @@ -959,6 +1310,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/ac/f9e4e731635192571f86f52d86234f537c7f8ca4f6917c56b29051c077ef/duckdb-1.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:a3be2072315982e232bfe49c9d3db0a59ba67b2240a537ef42656cc772a887c7", size = 14370790, upload-time = "2026-03-23T12:12:12.497Z" }, ] +[[package]] +name = "einops" +version = "0.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/77/850bef8d72ffb9219f0b1aac23fbc1bf7d038ee6ea666f331fa273031aa2/einops-0.8.2.tar.gz", hash = "sha256:609da665570e5e265e27283aab09e7f279ade90c4f01bcfca111f3d3e13f2827", size = 56261, upload-time = "2026-01-26T04:13:17.638Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/09/f8d8f8f31e4483c10a906437b4ce31bdf3d6d417b73fe33f1a8b59e34228/einops-0.8.2-py3-none-any.whl", hash = "sha256:54058201ac7087911181bfec4af6091bb59380360f069276601256a76af08193", size = 65638, upload-time = "2026-01-26T04:13:18.546Z" }, +] + +[[package]] +name = "email-validator" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dnspython" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/22/900cb125c76b7aaa450ce02fd727f452243f2e91a61af068b40adba60ea9/email_validator-2.3.0.tar.gz", hash = "sha256:9fc05c37f2f6cf439ff414f8fc46d917929974a82244c20eb10231ba60c54426", size = 51238, upload-time = "2025-08-26T13:09:06.831Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/15/545e2b6cf2e3be84bc1ed85613edd75b8aea69807a71c26f4ca6a9258e82/email_validator-2.3.0-py3-none-any.whl", hash = "sha256:80f13f623413e6b197ae73bb10bf4eb0908faf509ad8362c5edeb0be7fd450b4", size = 35604, upload-time = "2025-08-26T13:09:05.858Z" }, +] + [[package]] name = "executing" version = "2.2.1" @@ -980,6 +1353,149 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cc/9a/74db0cf3115df2f71edcaf8d86ed556195ac31575212c20425820f81bfd0/Faker-20.1.0-py3-none-any.whl", hash = "sha256:aeb3e26742863d1e387f9d156f1c36e14af63bf5e6f36fb39b8c27f6a903be38", size = 1739139, upload-time = "2023-11-20T18:09:14.773Z" }, ] +[[package]] +name = "fastapi" +version = "0.141.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "email-validator" }, + { name = "fastapi-cli", extra = ["standard"] }, + { name = "fastar" }, + { name = "httpx" }, + { name = "jinja2" }, + { name = "pydantic-extra-types" }, + { name = "pydantic-settings" }, + { name = "python-multipart" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[[package]] +name = "fastapi-cli" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "rich-toolkit" }, + { name = "typer" }, + { name = "uvicorn", extra = ["standard"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/eb/3b534c6f8e157f9ddbf2a153512307c886cad0b258739c200dd8ff8c4452/fastapi_cli-0.0.32.tar.gz", hash = "sha256:38024d2345275e1b37ce8848727a580d84901b570e96b3256d9d36a9a5039424", size = 26636, upload-time = "2026-07-16T12:16:58.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/53/56ae5ae17bb0a5d89d1d31e5320eb1865553ebbfbde91cdc4c221245f2a8/fastapi_cli-0.0.32-py3-none-any.whl", hash = "sha256:8dcc286fa32f01bbd3f65dd09cfd5a2540ed5f2230b77db7fd30978d6165f3c4", size = 14670, upload-time = "2026-07-16T12:16:57.297Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "fastapi-cloud-cli" }, + { name = "uvicorn", extra = ["standard"] }, +] + +[[package]] +name = "fastapi-cloud-cli" +version = "0.23.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "detect-installer" }, + { name = "fastar" }, + { name = "httpx" }, + { name = "pydantic", extra = ["email"] }, + { name = "rich-toolkit" }, + { name = "rignore" }, + { name = "sentry-sdk" }, + { name = "typer" }, + { name = "uvicorn", extra = ["standard"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/00/dc/63aaf9913f455e39a7027c27140edd887a87d47d65ac43532d77a51718e5/fastapi_cloud_cli-0.23.0.tar.gz", hash = "sha256:840895bb8d14309aeffc905e0dcd1334d18c6f5da54b735413a8f1cb385e581e", size = 95295, upload-time = "2026-07-28T14:03:33.463Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/96/7e9aba6fabce3cb05f320abeef5b81efd5134823ae85d1a517872cb83cbc/fastapi_cloud_cli-0.23.0-py3-none-any.whl", hash = "sha256:1cd2ffa56e92e92c1fc63acc426c214dd928cbeed2a4c7c6a9a5fc85ea73de16", size = 78058, upload-time = "2026-07-28T14:03:34.386Z" }, +] + +[[package]] +name = "fastar" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/03/0f/0aeb3fc50046617702acc0078b277b58367fd62eb727b9ec733ae0e8bbcc/fastar-0.11.0.tar.gz", hash = "sha256:aa7f100f7313c03fdb20f1385927ba95671071ba308ad0c1763fef295e1895ce", size = 70238, upload-time = "2026-04-13T17:11:17.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/7c/0ed6dd38b9adc04b3a8ec3b7045908e7c2170ba0ff6e6d2c51bc9fc770f3/fastar-0.11.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:a6931bebc1d8e95ddeef55732c195449e6b44ef33aa31b325505097ed3b4d6aa", size = 869663, upload-time = "2026-04-13T17:09:09.78Z" }, + { url = "https://files.pythonhosted.org/packages/58/ce/8b7fb3f23855accebaaf2d2637eac7f261a7a5d936f861a172079f1ef511/fastar-0.11.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:891f72ce42a5e28a74fbd4d5fbf1a3ac1a1163d13cbc200cbd005fb0fabc54bd", size = 762938, upload-time = "2026-04-13T17:07:54.51Z" }, + { url = "https://files.pythonhosted.org/packages/07/cc/5491e2b677bb841f768e3aba052d0344338a5c78aa5d4c18b443831a8e8d/fastar-0.11.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5b83c1f61f7017d6e1498568038f8745440cfc16ca2f697ec81bac83050108f6", size = 759232, upload-time = "2026-04-13T17:08:08.864Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b7/643630bdbd179e41e9fae31c03b4cf6061dbf4d6fbbae8425d16eb12545d/fastar-0.11.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db73a9b765a516e73983b25341e7b5e0189733878279e278b2295131b0e3a21e", size = 926271, upload-time = "2026-04-13T17:08:23.68Z" }, + { url = "https://files.pythonhosted.org/packages/09/5d/37ade50003b4540e0a53ef100f6692d7ab2ac1122d5acf39920cc09a3e8b/fastar-0.11.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:625827d52eb4e8fec942e0233f125ff8010fcf6a67c0a974a8e5f4666b771e3c", size = 818634, upload-time = "2026-04-13T17:08:54.268Z" }, + { url = "https://files.pythonhosted.org/packages/c3/ff/135d177de32cc1e837c99019e4643e6e79352bde49544d4ece5b5eebf56b/fastar-0.11.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7f5fd8fa21ec0a88296a38dc5d7fc35efd3b26d46a17b8b7c73c5563925ca15", size = 822755, upload-time = "2026-04-13T17:09:25.01Z" }, + { url = "https://files.pythonhosted.org/packages/27/cb/b835dbe76ceac7fa6105851468c259ffd06830eb9c029402e499d0ec153b/fastar-0.11.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8c15af91b8cd87ddf23ea55355ae513c1de3ab67178f26dad017c9e9c0af6096", size = 887101, upload-time = "2026-04-13T17:08:39.248Z" }, + { url = "https://files.pythonhosted.org/packages/9e/54/aa8289eb57fc550535470397cb051f5a58a7c89ca4de31d5502b916dd894/fastar-0.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:03a112395a8b0bff251423bd1564c012f0cc058ad8b6bd8fba96f3d7fc117e44", size = 973606, upload-time = "2026-04-13T17:10:10.98Z" }, + { url = "https://files.pythonhosted.org/packages/1f/fd/776d50a0897c01dc6bfd0926772ee913436fdae91b9affaf0a0cbd09f0a1/fastar-0.11.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f2994bb8f5f8c11eb12beae1e6e77a907173c9819236b8a4c8f0573652ceccce", size = 1036696, upload-time = "2026-04-13T17:10:28.502Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f1/cf0f9b499fb37ac065c8a01ec642f96a3c5eb849c38ae983b59f3b3245e0/fastar-0.11.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:dcf99e4b5973d842c7f19c776c3a83cdc0977d505edce6206438505c0456b517", size = 1078182, upload-time = "2026-04-13T17:10:45.318Z" }, + { url = "https://files.pythonhosted.org/packages/f8/9e/21e4701aec4a1123d4dc4d31578dc18875582b5710e4725f7ceb752a248b/fastar-0.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29c9c386dc0d5dda78845a8e6b1480d26ab861c1e0b68f42ae5735cb70ca07f1", size = 1032336, upload-time = "2026-04-13T17:11:02.364Z" }, + { url = "https://files.pythonhosted.org/packages/ab/69/9816d69ac8265c9e50456637a487ccfb7a9c566efd9dbcd673df9c2558c2/fastar-0.11.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bd2f05666d4df7e14885b5c38fefd92a785917387513d33d837ff42ec143a22f", size = 863950, upload-time = "2026-04-13T17:09:11.506Z" }, + { url = "https://files.pythonhosted.org/packages/5b/0d/f88daad53aff2e754b6b5ff2a7113f72447a34f6ef17cc23ca99988117b7/fastar-0.11.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c1e6e74aba1ae77ca4aedcaf1697cd413319f4c88a5ccbe5b42c709517c5097e", size = 760737, upload-time = "2026-04-13T17:07:55.958Z" }, + { url = "https://files.pythonhosted.org/packages/2f/a6/82ef4ecd969d50d92ed3ed9dbd8fe77faa24be5e5736f716edc9f4ce8d62/fastar-0.11.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:38ef77fe940bbc9b37a98bd838727f844b11731cd39358a2640ff864fb385086", size = 757603, upload-time = "2026-04-13T17:08:10.623Z" }, + { url = "https://files.pythonhosted.org/packages/03/35/50249f0d827251f8ac511495e2eacccebda80a00a0ad73e9615b8113b84f/fastar-0.11.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8955e61b32d6aff82c983217abf80933fd823b0e727586fc72f08043d996fd59", size = 923952, upload-time = "2026-04-13T17:08:25.526Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d8/faee41659e9c379d906d24eaee6d6833ac8cfef0a5df480e5c2a8d3efb33/fastar-0.11.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:483532442cdb08fbff0169510224eae0836f2f672cea6aacb52847d90fefdc46", size = 816574, upload-time = "2026-04-13T17:08:56.076Z" }, + { url = "https://files.pythonhosted.org/packages/22/47/0448ea7992b997dad2bf004bfd98eca74b5858630eae080b50c7b17d9ddc/fastar-0.11.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef5a6071121e05d8287fc75bccb054bcbac8bb0501200a0c0a8feeace5303ea4", size = 819382, upload-time = "2026-04-13T17:09:26.66Z" }, + { url = "https://files.pythonhosted.org/packages/33/ef/0d63eb43586831b7a6f8b22c4d77125a7c594423af1f4f090fa9541b9b40/fastar-0.11.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:e45e598af5afe8412197d4786efd6cf29be02e7d3d4f6a3461149eae5d7e94f1", size = 885254, upload-time = "2026-04-13T17:08:40.9Z" }, + { url = "https://files.pythonhosted.org/packages/01/25/edd584675d69e49a165052c3ee886df1c5d574f3e7d813c990306387c623/fastar-0.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2e160919b1c47ddb8538e7e8eb4cd527281b40f0bf75110a75993838ef61f286", size = 971239, upload-time = "2026-04-13T17:10:12.997Z" }, + { url = "https://files.pythonhosted.org/packages/a5/37/e8bb24f506ba2b08fbaf36c5800e843bd4d542954e9331f00418e2d23349/fastar-0.11.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:4bb4dc0fc8f7a6807febcebce8a2f3626ba4955a9263d81ecc630aad83be84c0", size = 1035185, upload-time = "2026-04-13T17:10:30.207Z" }, + { url = "https://files.pythonhosted.org/packages/9a/bf/be753736296338149ee4cb3e92e2b5423d6ba17c7b951d15218fd7e99bbf/fastar-0.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4ec95af56aa173f6e320e1183001bf108ba59beaf13edd1fc8200648db203588", size = 1072191, upload-time = "2026-04-13T17:10:47.072Z" }, + { url = "https://files.pythonhosted.org/packages/d2/cd/a81c1aaafb5a22ce57c98ae22f39c89413ed53e4ee6e1b1444b0bd666a6c/fastar-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:136cf342735464091c39dc3708168f9fdeb9ebea40b1ead937c61afaf46143d9", size = 1028054, upload-time = "2026-04-13T17:11:04.293Z" }, + { url = "https://files.pythonhosted.org/packages/e5/2b/d11d84bdd5e0e377771b955755771e3460b290da5809cb78c1b735ee2228/fastar-0.11.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:881247e6b6eaea59fc6569f9b61447aa6b9fc2ee864e048b4643d69c52745805", size = 863054, upload-time = "2026-04-13T17:09:13.048Z" }, + { url = "https://files.pythonhosted.org/packages/25/39/d3f428b318fa940b1b6e785b8d54fc895dfb5d5b945ef8d5442ffa904fb2/fastar-0.11.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:863b7929845c9fec92ef6c8d59579cf46af5136655e5342f8df5cebe46cab06c", size = 760247, upload-time = "2026-04-13T17:07:57.396Z" }, + { url = "https://files.pythonhosted.org/packages/9e/04/03949aee82aabb8ede06ac5a4a5579ffaf98a8fe59ce958494508ff15513/fastar-0.11.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:96b4a57df12bf3211662627a3ea29d62ecb314a2434a0d0843f9fc23e47536e5", size = 756512, upload-time = "2026-04-13T17:08:12.415Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0c/2ca1ae0a3828ca51047962d932b80daca2522db73e8cb9d040cb6ebe28d5/fastar-0.11.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ceef1c2c4df7b7b8ebd3f5d718bbf457b9bbdf25ce0bd07870211ec4fbd9aff4", size = 922183, upload-time = "2026-04-13T17:08:27.187Z" }, + { url = "https://files.pythonhosted.org/packages/65/68/7fe808b1f73a68e686f25434f538c6dc10ef4dfb3db0ace22cd861744bf8/fastar-0.11.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8e545918441910a779659d4759ad0eef349e935fbdb4668a666d3681567eb05", size = 816394, upload-time = "2026-04-13T17:08:57.657Z" }, + { url = "https://files.pythonhosted.org/packages/1f/17/07d086080f8a83b8d7966955e29bcdbd6a060f5bd949dc9d5abd3658cead/fastar-0.11.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28095bb8f821e85fc2764e1a55f03e5e2876dee2abe7cd0ee9420d929905d643", size = 818983, upload-time = "2026-04-13T17:09:28.46Z" }, + { url = "https://files.pythonhosted.org/packages/fb/e2/2c4edf0910af2e814ff6d65b77a91196d472ca8a9fb2033bd983f6856caa/fastar-0.11.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0fafb95ecbe70f666a5e9b35dd63974ccdc9bb3d99ccdbd4014a823ec3e659b5", size = 884689, upload-time = "2026-04-13T17:08:42.763Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/04fdcbd6558e60de4ced3b55230fac47675d181252582b2fcec3c74608e5/fastar-0.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:af48fed039b94016629dcdad1c95c90c486326dd068de2b0a4df419ee09b6821", size = 970677, upload-time = "2026-04-13T17:10:15.124Z" }, + { url = "https://files.pythonhosted.org/packages/df/b3/2b860a9658550167dbd5824c85e88d0b4b912bf493e42a6322544d6e483d/fastar-0.11.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:74cd96163f39b8638ab4e8d49708ca887959672a22871d8170d01f067319533b", size = 1034026, upload-time = "2026-04-13T17:10:32.318Z" }, + { url = "https://files.pythonhosted.org/packages/b7/9b/fa42ea1188b144bac4b1b60753dfd449974a4d5eda132029ee7711569f94/fastar-0.11.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:4e8b993cb5613bab495ed482810bedc0986633fcb9a3b55c37ec88e0d6714f6a", size = 1071147, upload-time = "2026-04-13T17:10:48.833Z" }, + { url = "https://files.pythonhosted.org/packages/95/c8/d2e501556dca9f1fbc9246111a31792fb49ad908fa4927f34938a97a3604/fastar-0.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:dfe39d91fc28e37e06162d94afe01050220edb7df554acb5b702b5503e564816", size = 1028377, upload-time = "2026-04-13T17:11:06.374Z" }, + { url = "https://files.pythonhosted.org/packages/dc/5e/9395c7353d079cb4f5be0f7982ce0dc9f2e7dec5fd175eef466729d6023a/fastar-0.11.0-cp314-cp314-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7c371f1d4386c699018bb64eb2fa785feacf32785559049d2bb72fe4af023f53", size = 864378, upload-time = "2026-04-13T17:09:14.611Z" }, + { url = "https://files.pythonhosted.org/packages/fa/ba/1e4f67148223ff219612b6281a6000357abbcc2417964fa5c83f11d68fce/fastar-0.11.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cad7fa41e3e66554387481c1a09365e4638becd322904932674159d5f4046728", size = 760921, upload-time = "2026-04-13T17:07:59.138Z" }, + { url = "https://files.pythonhosted.org/packages/0f/82/09d11fb6d12f17993ffaf32ffd30c3c121a11e2966e84f19fb6f66430118/fastar-0.11.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf36652fa71b83761717c9899b98732498f8a2cb6327ff16bbf07f6be85c3437", size = 757012, upload-time = "2026-04-13T17:08:14.186Z" }, + { url = "https://files.pythonhosted.org/packages/52/1f/5aeeacc4cb65615e2c9292cd9c5b0cd6fb6d2e6ee472ca6adc6c1b1b22ef/fastar-0.11.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f68ff8c17833053da4841720e95edde80ce45bb994b6b7d51418dddaac70ee47", size = 924510, upload-time = "2026-04-13T17:08:28.741Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1a/1e5bdabbeaf2e856928956292609f2ff6a650f94480fb8afaca30229e483/fastar-0.11.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4563ed37a12ea1cdc398af8571258d24b988bf342b7b3bf5451bd5891243280c", size = 816602, upload-time = "2026-04-13T17:08:59.461Z" }, + { url = "https://files.pythonhosted.org/packages/87/24/f960147910da3bed41a3adfcb026e17d5f50f4cf467a3324237a7088f61a/fastar-0.11.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cee63c9875cba3b70dc44338c560facc5d6e763047dcc4a30501f9a68cf5f890", size = 819452, upload-time = "2026-04-13T17:09:29.926Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f4/3e77d7901d5707fd7f8a352e153c8ae09ea974e6fabad0b7c4eb9944b8d4/fastar-0.11.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:bd76bfffae6d0a91f4ac4a612f721e7aec108db97dccdd120ae063cd66959f27", size = 885254, upload-time = "2026-04-13T17:08:44.285Z" }, + { url = "https://files.pythonhosted.org/packages/47/01/1585edd5ec47782ae93cd94edf05828e0ab02ef00aec00aea4194a600464/fastar-0.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8f5b707501ec01c1bc0518f741f01d322e50c9adc19a451aa24f67a2316e9397", size = 971496, upload-time = "2026-04-13T17:10:17.024Z" }, + { url = "https://files.pythonhosted.org/packages/f1/e9/6874c9d1236ded565a0bed54b320ac9f165f287b1d89490fb70f9f323c81/fastar-0.11.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:37c0b5a88a657839aad98b0a6c9e4ac4c2c15d6b49c44ee3935c6b08e9d3e479", size = 1034685, upload-time = "2026-04-13T17:10:34.063Z" }, + { url = "https://files.pythonhosted.org/packages/14/d8/4ab20613ce2983427aee958e39be878dba874aa227c530a845e32429c4f6/fastar-0.11.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:6c55f536c62a6efb180c1af0d5182948bff576bbfe6276e8e1359c9c7d2215d8", size = 1072675, upload-time = "2026-04-13T17:10:50.53Z" }, + { url = "https://files.pythonhosted.org/packages/1f/ae/5ac3b7c20ce4b08f011dd2b979f96caabe64f9b10b157f211ea91bdfadca/fastar-0.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3082eeca59e189b9039335862f4c2780c0c8871d656bfdf559db4414a105b251", size = 1029330, upload-time = "2026-04-13T17:11:08.138Z" }, + { url = "https://files.pythonhosted.org/packages/2b/c3/38f1dac77ae0c71c37b176277c96d830796b8ce2fe69705f917829b53829/fastar-0.11.0-cp314-cp314t-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:bd3eca3bbfec84a614bcb4143b4ad4f784d0895babc26cfc88436af88ca23c7a", size = 864403, upload-time = "2026-04-13T17:09:16.58Z" }, + { url = "https://files.pythonhosted.org/packages/6e/f0/e69c363bdb3e5a5848e937b662b5469581ee6682c51bc1c0556494773929/fastar-0.11.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ff86a967acb0d621dd24063dda090daa67bf4993b9570e97fe156de88a9006ca", size = 759480, upload-time = "2026-04-13T17:08:00.599Z" }, + { url = "https://files.pythonhosted.org/packages/3b/29/4d8737590c2a6357d614d7cc7288e8f68e7e449680b8922997cc4349e65e/fastar-0.11.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:86eaf7c0e985d93a7734168be2fb232b2a8cca53e41431c2782d7c12b12c03b1", size = 756219, upload-time = "2026-04-13T17:08:15.699Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ec/400de7b3b7d48801908f19cf5462177104395799472671b3e8152b2b04ca/fastar-0.11.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91f07b0b8eb67e2f177733a1f884edad7dfb9f8977ffef15927b20cb9604027d", size = 923669, upload-time = "2026-04-13T17:08:30.574Z" }, + { url = "https://files.pythonhosted.org/packages/5d/01/8926c53da923fed7ab4b96e7fbf7f73b663beb4f02095b654d6fab46f9ad/fastar-0.11.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f85c896885eb4abf1a635d54dea22cac6ae48d04fc2ea26ae652fcf1febe1220", size = 815729, upload-time = "2026-04-13T17:09:01.204Z" }, + { url = "https://files.pythonhosted.org/packages/89/f0/5fef4c7946e352651b504b1a4235dac3505e7cfd24020788ab50552e84bf/fastar-0.11.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:075c07095c8de4b774ba8f28b9c0a02b1a2cd254da50cbe464dd3bb2432e9158", size = 819812, upload-time = "2026-04-13T17:09:31.907Z" }, + { url = "https://files.pythonhosted.org/packages/b3/c8/0ebc3298b4a45e7bddc50b169ae6a6f5b80c939394d4befe6e60de535ee7/fastar-0.11.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:07f028933820c65750baf3383b807ecce1cd9385cf00ce192b79d263ad6b856c", size = 884074, upload-time = "2026-04-13T17:08:45.802Z" }, + { url = "https://files.pythonhosted.org/packages/ae/9f/7baa4cdff8d6fbca41fa5c764b48a941fed8a9ec6c4cc92de65895a28299/fastar-0.11.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:039f875efa0f01fa43c20bf4e2fc7305489c61d0ac76eda991acfba7820a0e63", size = 969450, upload-time = "2026-04-13T17:10:18.667Z" }, + { url = "https://files.pythonhosted.org/packages/d4/dc/1ebbfb58a47056ba866494f19efbcdd2ba2897096b94f36e796594b4d05b/fastar-0.11.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:fff12452a9a5c6814a012445f26365541cc3d99dcca61f09762e6a389f7a32ea", size = 1033775, upload-time = "2026-04-13T17:10:36.165Z" }, + { url = "https://files.pythonhosted.org/packages/c2/5f/ce4e3914066f08c99eb8c32952cc07c1a013e81b1db1b0f598130bf6b974/fastar-0.11.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:2bf733e09f942b6fa876efe30a90508d1f4caef5630c00fb2a84fba355873712", size = 1072158, upload-time = "2026-04-13T17:10:52.497Z" }, + { url = "https://files.pythonhosted.org/packages/03/2a/6bca72992c84151c387cc6558f3867f5ebe5fb3684ee6fa9b76280ba4b8e/fastar-0.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d1531fa848fdd3677d2dce0a4b436ea64d9ae38fb8babe2ddbc180dd153cb7a3", size = 1028577, upload-time = "2026-04-13T17:11:09.934Z" }, + { url = "https://files.pythonhosted.org/packages/7e/36/8d4569e26473c72ccb02d1c5df3ed710073f1c06eca09c26d52ea79fd815/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8800e2387e463a0e5799416a1cbe72dd0fde7270a20e4bde684145e7878f6516", size = 870850, upload-time = "2026-04-13T17:09:21.439Z" }, + { url = "https://files.pythonhosted.org/packages/bf/46/724dc796e1756d3977970f820d30d59bb8cab8e3671b285f1d82ab513aec/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7496def0a2befd82d429cb004ef7ca831585cc887947bd6b9abb68a5ef852b0b", size = 764469, upload-time = "2026-04-13T17:08:05.638Z" }, + { url = "https://files.pythonhosted.org/packages/99/e3/74d6859e632e8fb9339a14f652fb9f800c2bd6aa53071e311c0be3fbab8b/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:878eaf15463eb572e3538af7ca3a8534e5e279cf8196db902d24e5725c4af86e", size = 761375, upload-time = "2026-04-13T17:08:20.669Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e7/cc70e2be5ef8731a7525552b1c35c1448cf9eae6a62cb3a56f12c1bf27ea/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0324ed1d1ef0186e1bbd843b17807d6d837d0906899d4c99378b02c5d86bdd9c", size = 928189, upload-time = "2026-04-13T17:08:35.663Z" }, + { url = "https://files.pythonhosted.org/packages/3c/33/c9a969e78dca323547276a6fee5f4f9588f7cd5ab45acec3778c67399589/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bdf9bd863205590beaf8ef6e66f315310196632180dceaf674985d01a876cac3", size = 820864, upload-time = "2026-04-13T17:09:06.366Z" }, + { url = "https://files.pythonhosted.org/packages/84/bd/6b9434b541fe55c125b5f2e017a565596a2d215aa09207e4555e4585064f/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59af8dbb683b24b90fb5b506de080faeab0a17a908e6c2a5d93a97260ed75d7b", size = 824060, upload-time = "2026-04-13T17:09:37.377Z" }, + { url = "https://files.pythonhosted.org/packages/24/8d/871d5f8cf4c6f13987119fb0a9ae8be131e34f2756c2524e9974adf33824/fastar-0.11.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:9f3df73a3c4292cfe15696cdf59cdb6c309ab59d30b34c733be13c6e32d9a264", size = 889217, upload-time = "2026-04-13T17:08:50.884Z" }, + { url = "https://files.pythonhosted.org/packages/d0/26/cca0fd2704f3ed20165e5613ed911549aef3aaf3b0b5b02fee0e8e23e6cc/fastar-0.11.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:aa3762cbb16e41a76b61f4a6914937a71aab3a7b6c2d82ca233bc686ebaf756b", size = 975418, upload-time = "2026-04-13T17:10:24.307Z" }, + { url = "https://files.pythonhosted.org/packages/99/94/8bbb0b13f5b6cbe2492f0b7cbba5103e6163976a3331466d010e781fa189/fastar-0.11.0-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:a8c7bc8ac74cb359bb546b199288c83236372d094b402e557c197e85527495cd", size = 1038492, upload-time = "2026-04-13T17:10:41.939Z" }, + { url = "https://files.pythonhosted.org/packages/ed/d3/5b7df222a30eac2822ffd00f82fd4c2ce84fba4b369d1e1a03732fd177fc/fastar-0.11.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:587cbd060a2699c5f66281081395bb4657b2b1e0eef5c206b1aabf740019d670", size = 1080210, upload-time = "2026-04-13T17:10:58.462Z" }, + { url = "https://files.pythonhosted.org/packages/ec/6d/56ef943ea524784598c035ccbd42e564e937da0438ae3f55f0e76cb95571/fastar-0.11.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6a1c56957ac82408be37a3f63594bc83e0919e8760492a4475e542f9f1828778", size = 1034886, upload-time = "2026-04-13T17:11:15.617Z" }, +] + [[package]] name = "fastjsonschema" version = "2.21.2" @@ -989,6 +1505,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, ] +[[package]] +name = "fastsafetensors" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/4c/f17bd54c933fd23648ce1e4272adf27d8d7e99b788c8859544bbd39f02e7/fastsafetensors-0.3.3.tar.gz", hash = "sha256:ba4fb59be8a6adbc91723848c3c6f57a9a9a5d2247d9768f84511380d01e554c", size = 77817, upload-time = "2026-07-07T07:21:47.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/51/40bdc05f922251c54518232b2ef47fd925094ee699c93a45c8792628a10e/fastsafetensors-0.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df2f641647b79093c5e3f4bb0b9c7d771f22b61279714c3059d5c71a7cf708da", size = 1858970, upload-time = "2026-07-07T07:21:30.457Z" }, + { url = "https://files.pythonhosted.org/packages/2f/f5/79f187385b4b9093742fec94e7889d22634ab6eba8403707797246fb2418/fastsafetensors-0.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92f8cf8e6617cffe24522023c726c5911c1ca5b7e3249da054abb0fac8d27040", size = 1891339, upload-time = "2026-07-07T07:21:32.022Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/f95f7fc099ac1fc4c22aa46257d159eac88e29dd0765d21c4fc91caedb01/fastsafetensors-0.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c2f788a936ffd17938484360339645812e14cf1b2bdf4c18c035c713218e5a9", size = 1887326, upload-time = "2026-07-07T07:21:34.624Z" }, + { url = "https://files.pythonhosted.org/packages/92/8c/e3347b2a44a8ab9aced94fa450df4f309baa21f7f2981a8a7bd6a977f4d3/fastsafetensors-0.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3587bc66b8dec560ad903becf9540889013d4d47f0e10cf45f19bedc7b7bffa7", size = 1915478, upload-time = "2026-07-07T07:21:35.901Z" }, + { url = "https://files.pythonhosted.org/packages/16/a2/fd30fe6ed5825cb4642bdd102d3c5a32d2d82c91bdb45ae61a9571278cf8/fastsafetensors-0.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f969b2c818942748ff1bc619e6d575a7a7156a7d488d4c41779fba8c4e1891dc", size = 1887148, upload-time = "2026-07-07T07:21:38.414Z" }, + { url = "https://files.pythonhosted.org/packages/d6/9d/d3db53dd5b5069b44f2b8f6fc5a943f4101178a6f74b48a7fc72dc68f142/fastsafetensors-0.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbf9fbc5f74d6a3f333777eb222aa63453795dacd7ae7f559316a5398d4942ef", size = 1915579, upload-time = "2026-07-07T07:21:40.047Z" }, + { url = "https://files.pythonhosted.org/packages/24/80/aeb98bc1575cbd759466568ac6dcbad0a572622cdee0a604fb9604577ecf/fastsafetensors-0.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e3386834e1595834124cfbb23e05a732cb1b4fb5f98562375c1806f14fe1d677", size = 1885958, upload-time = "2026-07-07T07:21:42.812Z" }, + { url = "https://files.pythonhosted.org/packages/1a/16/dc09db973bdc763d5ff64a36acc9e847d0b6c150591d9917ed65f304b722/fastsafetensors-0.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7db5f3ee98433cc8e5dcaaef2517f47c34a51ea8229350406b24b2f84af209d3", size = 1913491, upload-time = "2026-07-07T07:21:44.32Z" }, +] + [[package]] name = "filelock" version = "3.20.3" @@ -998,6 +1533,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, ] +[[package]] +name = "flashinfer-cubin" +version = "0.6.8.post1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/b7/5e3b1a8c67031b421a8bd29c2bc29b900a550bb3392e8bda18bb15b5e476/flashinfer_cubin-0.6.8.post1-py3-none-any.whl", hash = "sha256:43636d4cd39e694a83d76a89f87fefcdf4cecb4c4f7dd22dac25ec368c1e901f", size = 295154113, upload-time = "2026-04-18T18:28:21.738Z" }, +] + +[[package]] +name = "flashinfer-python" +version = "0.6.8.post1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi" }, + { name = "click" }, + { name = "cuda-tile" }, + { name = "einops" }, + { name = "ninja" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "nvidia-cudnn-frontend" }, + { name = "nvidia-cutlass-dsl" }, + { name = "nvidia-ml-py" }, + { name = "packaging" }, + { name = "requests" }, + { name = "tabulate" }, + { name = "torch" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/1e/2760fef9e74abc4480961048e5790b4c9e955872fb4d7d97900cfddced5a/flashinfer_python-0.6.8.post1.tar.gz", hash = "sha256:b18e4121baf9b93fa9a9f368ba9b981a0342895f50ab9dddc224aeb964ed346f", size = 6675885, upload-time = "2026-04-18T18:28:13.299Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/6d/1e8a8533913e33a50a486332ce0673f4fdb860f6eb9ed450327c5c1762cb/flashinfer_python-0.6.8.post1-py3-none-any.whl", hash = "sha256:818f9b8cc2fe66c42a1f6264be4841ac8821ada703685a02cfccb2b5124a710b", size = 9385316, upload-time = "2026-04-18T18:28:10.285Z" }, +] + [[package]] name = "fqdn" version = "1.5.1" @@ -1126,6 +1695,22 @@ http = [ { name = "aiohttp" }, ] +[[package]] +name = "gguf" +version = "0.19.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/ae/17f1308ae45cd7b08ebb521747d5b23f4efc4d172038a4e228dd5106c3ff/gguf-0.19.0.tar.gz", hash = "sha256:dbadcd6cc7ccd44256f2229fe7c2dff5e8aa5cf0612ab987fd2b1a57e428923f", size = 111220, upload-time = "2026-05-06T13:04:03.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/bb/d71d6da82763528c2c2ed6b59a9d6142c6595545a4c448e2085d155e88c2/gguf-0.19.0-py3-none-any.whl", hash = "sha256:70bcd10edfe697fb2dad6e40af2234b9d8ece9a41a99761405121ebda1c3c1cd", size = 118475, upload-time = "2026-05-06T13:04:02.588Z" }, +] + [[package]] name = "ghp-import" version = "2.1.0" @@ -1162,6 +1747,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.75.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/73/74bcab964c9a7a61f2bb71e8179b0f13e6fa98f7ce00fd168aab291e4a2e/googleapis_common_protos-1.75.1.tar.gz", hash = "sha256:d3042c6c5a2d4e67113104d6b6818b59b6bd92a197f2a91508e801fe815cf071", size = 150967, upload-time = "2026-08-06T06:24:51.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/51/186c02b8549b69ccda44429cf6ff5081e4b61a602ddfe6a8020d1be31d1b/googleapis_common_protos-1.75.1-py3-none-any.whl", hash = "sha256:28a1934bcd33b9c9da66ac301a0a4227e3367f095a17d0375cb98f0a09d93b79", size = 300626, upload-time = "2026-08-06T06:23:46.696Z" }, +] + [[package]] name = "griffe-pydantic" version = "1.3.1" @@ -1183,6 +1780,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/51/c936033e16d12b627ea334aaaaf42229c37620d0f15593456ab69ab48161/griffelib-2.0.0-py3-none-any.whl", hash = "sha256:01284878c966508b6d6f1dbff9b6fa607bc062d8261c5c7253cb285b06422a7f", size = 142004, upload-time = "2026-02-09T19:09:40.561Z" }, ] +[[package]] +name = "grpcio" +version = "1.83.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0c/98/304898ac4e04e2d5e4e4c2eadc178b1f2a16d5f4bc2f91306c87d64680b9/grpcio-1.83.0.tar.gz", hash = "sha256:7674587248fbbb2ac6e4eecf83a8a0f3d91a928f941de571acfd3a2f007fbc24", size = 13428824, upload-time = "2026-07-23T15:20:37.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/f6/3b781cd07a715ea5f5125ae264226e7fc4d87603d6d3955022cabfdc5da2/grpcio-1.83.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:8ff0b8767ddd62704e0d9571c1890af08d84a3a689ebba1807e62519d0b3277f", size = 6338720, upload-time = "2026-07-23T15:19:13.177Z" }, + { url = "https://files.pythonhosted.org/packages/6b/98/8acbb416544e7871132d8e42a07ed70c802d70e6a16c6009e505a34d32a4/grpcio-1.83.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f4cee5fc86e84a0cf7ad1574b454c3320e087c07f55b7df5dc0ac6a873fb90c0", size = 6921203, upload-time = "2026-07-23T15:19:17.824Z" }, + { url = "https://files.pythonhosted.org/packages/45/9c/0fdbfaf4fc54e5c88f6bce4008a065092fe7fbc4460eb5617ae8b20fd505/grpcio-1.83.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f5e822a7e7d03282f6ad225e710493c48b9057a353358344a5f7c42b2b37618d", size = 7648508, upload-time = "2026-07-23T15:19:19.685Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ea/107b9dbb2ed3ad14dd774fd3dde7d29ff9938a6c198654becb2c3a0e9a6a/grpcio-1.83.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f5f410d7c2903eabb34789dfd6342eef04af1ad459943936b7e09a9f5bd417b9", size = 7079466, upload-time = "2026-07-23T15:19:21.478Z" }, + { url = "https://files.pythonhosted.org/packages/3b/06/9fa9941089e6fae83b060b6ce61c1e81053e52decae43197245f45e07d36/grpcio-1.83.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee94a4016fdf8699fb1fd8a38652475ff677f1c72074cee44deeeb9a7e95e745", size = 7605583, upload-time = "2026-07-23T15:19:23.74Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/f10fb56062dc2771c630827a82d9ad0ecd05cad572ea3b08d49f6631680a/grpcio-1.83.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:c6444666317338e903093c7c756e6cc88eee59f798cb8dd41e87725bf54e1617", size = 8637810, upload-time = "2026-07-23T15:19:25.536Z" }, + { url = "https://files.pythonhosted.org/packages/99/55/f84927258f6a1b6ea6dea661fdc6de859b35e560c96f3012d15ccd39f85e/grpcio-1.83.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:aa074041231f03959cb097dd5517b0677b8ea49215bae01d5710a7b69dd59969", size = 8008021, upload-time = "2026-07-23T15:19:27.863Z" }, + { url = "https://files.pythonhosted.org/packages/15/2b/51e32514a4e9b715375c99721aadff0f24164cc2049b8269eda4de82a814/grpcio-1.83.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:28f6c35ac8fcf10e4594f138e468f194360089dde40d126a7033e863fc479930", size = 6303167, upload-time = "2026-07-23T15:19:33.78Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5f/734e72e7b9f79bcf0b2c270b8d3bca0e4ebb97a27a50d06240b145f6d41e/grpcio-1.83.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6fb8a1dd0c6f0f931e69e9d0dc6d1c406ed2a44fa963414eafba07b7fb685d16", size = 6869310, upload-time = "2026-07-23T15:19:38.607Z" }, + { url = "https://files.pythonhosted.org/packages/a4/17/a1735f215b2a5cd43c38b79eac072ad197e61be9829905b6b29550abd0db/grpcio-1.83.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2b5e75c34842cd9c1b95285ca395c6a569664b81e3ffa6b714125922942abaaf", size = 7613472, upload-time = "2026-07-23T15:19:40.645Z" }, + { url = "https://files.pythonhosted.org/packages/b2/78/c9e81f806ac704b6b145cb01628db398985b1f8dfdc10e23b55fb0902b3d/grpcio-1.83.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeb339838db07600481ef869507279b75326c75eac6d10f7afa62a0da1d2bcdd", size = 7040616, upload-time = "2026-07-23T15:19:42.349Z" }, + { url = "https://files.pythonhosted.org/packages/9a/ba/94cd5af859876049d340480acbb61a959096c84b567f215534faa78d0424/grpcio-1.83.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:f47d62808b4c0a97b78bff88a6d4ca283a2a492b9a04a87d814af95ca3b9c19c", size = 7570491, upload-time = "2026-07-23T15:19:44.357Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/108d30d5a5c964312ae8b9cb0e8cc5b3c1cc68d8f757cca52b3565534d26/grpcio-1.83.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62003babc444a606dcd1f009cd16391ce23669ae4ad6ec267a873da7937a69f5", size = 8605036, upload-time = "2026-07-23T15:19:46.454Z" }, + { url = "https://files.pythonhosted.org/packages/ea/23/3828ae13c3db8233d123ad612747665817b952d8a954f32390230b582336/grpcio-1.83.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1aa567f8c3f19850ffd5d2858c9a8ea7c80f0db6c01186b71eb31e923ec984f5", size = 7981587, upload-time = "2026-07-23T15:19:48.913Z" }, + { url = "https://files.pythonhosted.org/packages/56/eb/135daaa713f32d33b8f99b4153b3f8dc3b2a124996ac15581bf9ebdad3c3/grpcio-1.83.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:6662f3b1e07cc7493d437351860dc867bddc6a93c83ecf33bbfdaf0c217ab2d0", size = 6304480, upload-time = "2026-07-23T15:19:53.962Z" }, + { url = "https://files.pythonhosted.org/packages/b0/e8/d0389e09cd6b4c4d3089b92967ae4e3ffd64795bd349bf2f85cd6656d3da/grpcio-1.83.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10b3fa0475eb572c9a81a6fe37fa16a9c500c0c91cfc148cac15692b7e3c2867", size = 6873200, upload-time = "2026-07-23T15:19:58.701Z" }, + { url = "https://files.pythonhosted.org/packages/f8/51/f464c1d211fa50d5adbabe1b2e519948d99c13757052bfc9ea7afa28e284/grpcio-1.83.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:5f20a988480b0f28207f057f7f7ae1313393c3cef0adcfeae8248f9947eaf881", size = 7618811, upload-time = "2026-07-23T15:20:00.733Z" }, + { url = "https://files.pythonhosted.org/packages/e8/c0/539fe0832f2dd6500a28f5263071623fb34e8d4867aec632ccf81bd21156/grpcio-1.83.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bd82671b39065ba18cd536e9cd45b27ff649053f81ddd2c6a966d595067080f", size = 7042310, upload-time = "2026-07-23T15:20:02.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ca/ccf617d37ffa72567fa8e005ec7090c99da922799be2fb9847c8b21ca18c/grpcio-1.83.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc60215b5cb9fc8ca72942c498b551ac2305bd08f6ef8d4e3f0d21b64fbecd61", size = 7575412, upload-time = "2026-07-23T15:20:04.712Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b9/fd8d5245f823a8e0fd35d90e20ea3aa4acd47f8d5318fa8df307df52dec6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f1c3e5689d4b90987b1d72022bcfe866a9a3dc66197484cf856d96b6150e7f45", size = 8604248, upload-time = "2026-07-23T15:20:06.77Z" }, + { url = "https://files.pythonhosted.org/packages/14/1e/f37632fc11db72dfa4bba86c3a43e54358e53030df111ecae5e91a733ad6/grpcio-1.83.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a21cb4eeeba124443f399be2e8b624943cde864dcbe588cb42e5c483a52a906c", size = 7977458, upload-time = "2026-07-23T15:20:09.109Z" }, + { url = "https://files.pythonhosted.org/packages/9c/60/f2cca8147ea213d3e43ae9158d03ad04e020fdf32ff027253e1fe93f921d/grpcio-1.83.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3f351629f6ae16ecc0ec3553e586a6763ffd9f6114044286d0cbec3e09241bfa", size = 6305607, upload-time = "2026-07-23T15:20:15.353Z" }, + { url = "https://files.pythonhosted.org/packages/92/ff/6f18f9426b69306f4e00a9add3b0ee2748da8aad53836ef80cab0d62d04f/grpcio-1.83.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7936f2a56cf04f6514705c0fedf400971de01b6aa1719327e4718f410a765e2b", size = 6880213, upload-time = "2026-07-23T15:20:19.98Z" }, + { url = "https://files.pythonhosted.org/packages/70/21/706d1147c6b93b98f179240c13991fbcc56880eba0c868abb1ad40d8a0a6/grpcio-1.83.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:b0a0be840e51b6b7ee9df9269770faf77bdf4b771053c257c21d12bad607714c", size = 7618335, upload-time = "2026-07-23T15:20:22.161Z" }, + { url = "https://files.pythonhosted.org/packages/74/04/1a8443c889115ec9e213a213e86bc93a71ee9088027e5befa09aaa0edd9d/grpcio-1.83.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:009667eaf3dcd5224c713589cdc98e7ca4ed0ff0b61132c6b276e930eb83a2df", size = 7043416, upload-time = "2026-07-23T15:20:24.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/c6/94e0fee5b12bc1da1370185b680988db6f739d19b42d9959db01a7ea50bf/grpcio-1.83.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bb669918fd88936b15599caff4160a77ab74bdeb25f2231f6e45b61282d6107b", size = 7583253, upload-time = "2026-07-23T15:20:26.313Z" }, + { url = "https://files.pythonhosted.org/packages/a0/97/de1ccb671fb85575bc5192faedf9ecdbdf5b390d2e6584dcf552bcbd370e/grpcio-1.83.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:c19b454d3d3f28db81f2c7c4dbaee96e7f6fd149721733ffe79d6bc530f17404", size = 8605102, upload-time = "2026-07-23T15:20:28.437Z" }, + { url = "https://files.pythonhosted.org/packages/17/0f/0e0ec749a7034ffcbaa050e39779872950ead90c22e7e0116be3f28b2b46/grpcio-1.83.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:61007cd08640abc5c54547ee32505474c482cd733a53cb87551ea81faa6350af", size = 7979826, upload-time = "2026-07-23T15:20:31.182Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1194,34 +1830,26 @@ wheels = [ [[package]] name = "hf-xet" -version = "1.5.0" +version = "1.6.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/d8/5c06fc76461418326a7decf8367480c35be11a41fd938633929c60a9ec6b/hf_xet-1.5.0.tar.gz", hash = "sha256:e0fb0a34d9f406eed88233e829a67ec016bec5af19e480eac65a233ea289a948", size = 837196, upload-time = "2026-05-06T06:18:15.583Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/68/9b/6912c99070915a4f28119e3c5b52a9abd1eec0ad5cb293b8c967a0c6f5a2/hf_xet-1.5.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:7d70fe2ce97b9db73b9c9b9c81fe3693640aec83416a966c446afea54acfae3c", size = 4023383, upload-time = "2026-05-06T06:17:53.947Z" }, - { url = "https://files.pythonhosted.org/packages/0f/6d/9563cfde59b5d8128a9c7ec972a087f4c782e4f7bac5a85234edfd5d5e49/hf_xet-1.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:73a0dae8c71de3b0633a45c73f4a4a5ed09e94b43441d82981a781d4f12baa42", size = 3792751, upload-time = "2026-05-06T06:17:51.791Z" }, - { url = "https://files.pythonhosted.org/packages/07/a5/ed5a0cf35b49a0571af5a8f53416dad1877a718c021c9937c3a53cb45781/hf_xet-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a60290ec57e9b71767fba7c3645ddafdd0759974b540441510c629c6db6db24a", size = 4456058, upload-time = "2026-05-06T06:17:40.735Z" }, - { url = "https://files.pythonhosted.org/packages/60/fb/3ae8bf2a7a37a4197d0195d7247fd25b3952e15cb8a599e285dfaa6f52b3/hf_xet-1.5.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e5de0f6deada0dada870bb376a11bcd1f08abf3a968a6d118f33e72d1b1eb480", size = 4250783, upload-time = "2026-05-06T06:17:38.412Z" }, - { url = "https://files.pythonhosted.org/packages/a2/9b/8bae40d4d91525085137196e84eb0ed49cf65b5e96e5c3ecdadd8bd0fac2/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c799d49f1a5544a0ef7591c0ee75e0d6b93d6f56dc7a4979f59f7518d2872216", size = 4445594, upload-time = "2026-05-06T06:18:04.219Z" }, - { url = "https://files.pythonhosted.org/packages/13/59/c74efbbd4e8728172b2cc72a2bc014d2947a4b7bdced932fbd3f5da1a4e5/hf_xet-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2baea1b0b989e5c152fe81425f7745ddc8901280ba3d97c98d8cdece7b706c60", size = 4663995, upload-time = "2026-05-06T06:18:06.1Z" }, - { url = "https://files.pythonhosted.org/packages/73/32/8e1e0410af64cda9b139d1dcebdc993a8ff9c8c7c0e2696ae356d75ccc0d/hf_xet-1.5.0-cp313-cp313t-win_amd64.whl", hash = "sha256:526345b3ed45f374f6317349df489167606736c876241ba984105afe7fd4839d", size = 3966608, upload-time = "2026-05-06T06:18:19.74Z" }, - { url = "https://files.pythonhosted.org/packages/fc/34/a8febc8f4edbea8b3e21b02ebc8b628679b84ba7e45cde624a7736b51500/hf_xet-1.5.0-cp313-cp313t-win_arm64.whl", hash = "sha256:786d28e2eb8315d5035544b9d137b4a842d600c434bb91bf7d0d953cce906ad4", size = 3796946, upload-time = "2026-05-06T06:18:17.568Z" }, - { url = "https://files.pythonhosted.org/packages/2a/20/8fc8996afe5815fa1a6be8e9e5c02f24500f409d599e905800d498a4e14d/hf_xet-1.5.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:872d5601e6deea30d15865ede55d29eac6daf5a534ab417b99b6ef6b076dd96c", size = 4023495, upload-time = "2026-05-06T06:18:01.94Z" }, - { url = "https://files.pythonhosted.org/packages/32/6a/93d84463c00cecb561a7508aa6303e35ee2894294eac14245526924415fe/hf_xet-1.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9929561f5abf4581c8ea79587881dfef6b8abb2a0d8a51915936fc2a614f4e73", size = 3792731, upload-time = "2026-05-06T06:18:00.021Z" }, - { url = "https://files.pythonhosted.org/packages/9d/5a/8ec8e0c863b382d00b3c2e2af6ded6b06371be617144a625903a6d562f4b/hf_xet-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f7b7bbae318e583a86fb21e5a4a175d6721d628a2874f4bd022d0e660c32a682", size = 4456738, upload-time = "2026-05-06T06:17:49.574Z" }, - { url = "https://files.pythonhosted.org/packages/c5/ca/f7effa1a67717da2bcc6b6c28f71c6ca648c77acaec4e2c32f40cbe16d85/hf_xet-1.5.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:cf7b2dc6f31a4ea754bb50f74cde482dcf5d366d184076d8530b9872787f3761", size = 4251622, upload-time = "2026-05-06T06:17:47.096Z" }, - { url = "https://files.pythonhosted.org/packages/65/f2/19247dba3e231cf77dec59ddfb878f00057635ff773d099c9b59d37812c3/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8dbcbab554c9ef158ef2c991545c3e970ddd8cc7acdcd0a78c5a41095dab4ded", size = 4445667, upload-time = "2026-05-06T06:18:11.983Z" }, - { url = "https://files.pythonhosted.org/packages/7f/64/6f116801a3bcfb6f59f5c251f48cadc47ea54026441c4a385079286a94fa/hf_xet-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5906bf7718d3636dc13402914736abe723492cb730f744834f5f5b67d3a12702", size = 4664619, upload-time = "2026-05-06T06:18:13.771Z" }, - { url = "https://files.pythonhosted.org/packages/5c/e8/069542d37946ed08669b127e1496fa99e78196d71de8d41eda5e9f1b7a58/hf_xet-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5f3dc2248fc01cc0a00cd392ab497f1ca373fcbc7e3f2da1f452480b384e839e", size = 3966802, upload-time = "2026-05-06T06:18:28.162Z" }, - { url = "https://files.pythonhosted.org/packages/f9/91/fc6fdec27b14d04e88c386ac0a0129732b53fa23f7c4a78f4b83a039c567/hf_xet-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b285cea1b5bab46b758772716ba8d6854a1a0310fed1c249d678a8b38601e5a0", size = 3797168, upload-time = "2026-05-06T06:18:26.287Z" }, - { url = "https://files.pythonhosted.org/packages/3d/fb/69ff198a82cae7eb1a69fb84d93b3a3e4816564d76817fe541ddc96874eb/hf_xet-1.5.0-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dad0dc84e941b8ba3c860659fe1fdc35c049d47cce293f003287757e971a8f56", size = 4030814, upload-time = "2026-05-06T06:17:57.933Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ff/edcc2b40162bef3ff78e14ab637e5f3b89243d6aee72f5949d3bb6a5af83/hf_xet-1.5.0-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a", size = 3798444, upload-time = "2026-05-06T06:17:55.79Z" }, - { url = "https://files.pythonhosted.org/packages/49/4d/103f76b04310e5e57656696cc184690d20c466af0bca3ca88f8c8ea5d4f3/hf_xet-1.5.0-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3531b1823a0e6d77d80f9ed15ca0e00f0d115094f8ac033d5cae88f4564cc949", size = 4465986, upload-time = "2026-05-06T06:17:44.886Z" }, - { url = "https://files.pythonhosted.org/packages/c4/a2/546f47f464737b3edbab6f8ddb57f2599b93d2cbb66f06abb475ccb48651/hf_xet-1.5.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:9a0ee58cd18d5ea799f7ed11290bbccbe56bdd8b1d97ca74b9cc49a3945d7a3b", size = 4259865, upload-time = "2026-05-06T06:17:42.639Z" }, - { url = "https://files.pythonhosted.org/packages/95/7f/1be593c1f28613be2e196473481cd81bfc5910795e30a34e8f744f6cac4f/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18", size = 4459835, upload-time = "2026-05-06T06:18:08.026Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b2/703569fc881f3284487e68cda7b42179978480da3c438042a6bbbb4a671c/hf_xet-1.5.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4b35549ce62601b84da4ff9b24d970032ace3d4430f52d91bcbb26c901d6c690", size = 4672414, upload-time = "2026-05-06T06:18:09.864Z" }, - { url = "https://files.pythonhosted.org/packages/af/37/1b6def445c567286b50aa3b33828158e135b1be44938dde59f11382a500c/hf_xet-1.5.0-cp37-abi3-win_amd64.whl", hash = "sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4", size = 3977238, upload-time = "2026-05-06T06:18:23.621Z" }, - { url = "https://files.pythonhosted.org/packages/62/94/3b66b148778ee100dcfd69c2ca22b57b41b44d3063ceec934f209e9184ce/hf_xet-1.5.0-cp37-abi3-win_arm64.whl", hash = "sha256:b6c9df403040248c76d808d3e047d64db2d923bae593eb244c41e425cf6cd7be", size = 3806916, upload-time = "2026-05-06T06:18:21.7Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/1b/ab/522a2ab67f27971a9d48ca666d4fca85ef7d5282d142e31fd087e27b1bbe/hf_xet-1.6.0.tar.gz", hash = "sha256:2e58454a340b3556dfa4972d5451aff4fba8dd42a236600ba1a1d2b1514f0fef", size = 920527, upload-time = "2026-08-03T22:33:13.243Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/62/3c062f593bd92ef4e77a0ef39541e3d82a0a1d3947c8a777a02a13a27828/hf_xet-1.6.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:70cbb9c896901600128cb9b6f06e132954fbede1db30f31f7c6c63f84cb7c31d", size = 4074584, upload-time = "2026-08-03T22:32:47.364Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1e/c0ad437dd267a8e435bef594acf781bbc3874ff0b6435b4962d03ecf7cc4/hf_xet-1.6.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:23379c2f9ec8696d952b16414a2bae72cad86a52df869b050698ba60f538c675", size = 3867381, upload-time = "2026-08-03T22:32:49.049Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ee/7c0d7b6ab336167531b1c30af2af003f054af4c749becbd7209ae33a77c3/hf_xet-1.6.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f7278c05c22fd60cb436cda1269649b3e81db65ecdc8496e5e164aa4143e7b", size = 4453982, upload-time = "2026-08-03T22:32:50.568Z" }, + { url = "https://files.pythonhosted.org/packages/63/06/ad8eab1c9525246650cbaa821caa3cdbaca734ab1a5b8c91bea09cbd8d69/hf_xet-1.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:948f15d3a9545cfe5932f6bd8b440f6ae630aee108f14b7bd6c561f7c2dcc522", size = 4249445, upload-time = "2026-08-03T22:32:52.391Z" }, + { url = "https://files.pythonhosted.org/packages/d8/26/1eee8aedb0dafc1ab9717dc9ac602cde33361b232dc06803f1f6ed18b58c/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5153e6bb103ad49d6ea9f1b2e230db5a2ea32551ad09a706d2f61d7c7c80d80e", size = 4451099, upload-time = "2026-08-03T22:32:54.114Z" }, + { url = "https://files.pythonhosted.org/packages/67/57/0b88af1f194ab6c9c650547d9cc06bfeaab836ae4dcdb331676bfb8be95a/hf_xet-1.6.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:35cec30d75c6f9eb9c16a77cef68e85a103b72e24d4b473714ec9ff06428bab9", size = 4664712, upload-time = "2026-08-03T22:32:55.547Z" }, + { url = "https://files.pythonhosted.org/packages/53/a0/26b717a9d1840e8abf48dcec64b5ed8fbe472671d38ad28d30e147132b33/hf_xet-1.6.0-cp314-cp314t-win_amd64.whl", hash = "sha256:5789835d7c6bc9436962853192082374297fb72d7eff7e7762ec25ceb7e25338", size = 4025906, upload-time = "2026-08-03T22:32:57.391Z" }, + { url = "https://files.pythonhosted.org/packages/49/f6/4a9966633c6fef83af997e2cff68ec1963676d412bdfd096df2a93b8e185/hf_xet-1.6.0-cp314-cp314t-win_arm64.whl", hash = "sha256:75765820ce4700db3750c94acc8fe27c5fae4c9ec000a0dbac3ca082acf97765", size = 3849221, upload-time = "2026-08-03T22:32:59.123Z" }, + { url = "https://files.pythonhosted.org/packages/a2/50/7afa2c9c787405864fc47a0d1bbc02c62e9101947ed43c1f43899fc7d91d/hf_xet-1.6.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:633dc0cd71d32da58ab8c03ad38e2fac452c15c2b0a2866ebf6ededfe0a5061d", size = 4071729, upload-time = "2026-08-03T22:33:00.721Z" }, + { url = "https://files.pythonhosted.org/packages/4b/69/55b8dcf636142ae660fec1869fcac14c4da2e8412e14d6eee1523be77e9f/hf_xet-1.6.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:f0906082d9932ae0c0057fa194041c22b4e2cdb46b2592ef3b91f020d62a081a", size = 3876287, upload-time = "2026-08-03T22:33:02.251Z" }, + { url = "https://files.pythonhosted.org/packages/67/4e/a28359bf1c1ecf11eba22123168c138698f7cb576ac678f5a2e16cd5da08/hf_xet-1.6.0-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d62671bb130879cef0ee4c9ebe47a14af6c66ec53e6d84dc15936e5ffdfac82f", size = 4464663, upload-time = "2026-08-03T22:33:03.802Z" }, + { url = "https://files.pythonhosted.org/packages/9a/69/1f0cbc2fb22ae6082d094f743d1b8945a3f36f6089cb95f42b7ee348cda7/hf_xet-1.6.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:0e6e21fa3cdfcdcd76748564bf593870a5e013f47d97cf10aed63aa222cff5b7", size = 4262538, upload-time = "2026-08-03T22:33:05.287Z" }, + { url = "https://files.pythonhosted.org/packages/d1/3a/4f4f2301ade26e404462d3336fa11f7958d914cabbabdd6e03c3c5d5658c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4fc74352a17015bd0ee90038bc9efe38db894cde45f268b6712b04fce8cd0acb", size = 4460520, upload-time = "2026-08-03T22:33:06.81Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/311725e2a905534dfee2dcb5b08414f249147f1f12252bfc2bd24caa075c/hf_xet-1.6.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4f71cba6129110c3374a33f919001ff130488fc23553698e34cc1c2a1198c", size = 4675937, upload-time = "2026-08-03T22:33:08.616Z" }, + { url = "https://files.pythonhosted.org/packages/98/b7/8c59a66d15205024662f1d66968136f13893f96df1ddc5087e2e281fc95f/hf_xet-1.6.0-cp38-abi3-win_amd64.whl", hash = "sha256:fb4fadde1b2b70bf4c0c14a6dccbe7194b1c28947fefd5bbe3fed9d940676c3b", size = 4033128, upload-time = "2026-08-03T22:33:10.171Z" }, + { url = "https://files.pythonhosted.org/packages/73/63/ca511b6f802f28cf3489b280fe77475bcca8de85e81a6299d7916b5b5555/hf_xet-1.6.0-cp38-abi3-win_arm64.whl", hash = "sha256:3dc3e35441ba395006af5aaacc40ef2e603c51ef46c3530b9156185f00935ea3", size = 3859359, upload-time = "2026-08-03T22:33:11.725Z" }, ] [[package]] @@ -1238,18 +1866,59 @@ wheels = [ ] [[package]] -name = "httpx" -version = "0.28.1" +name = "httpcore2" +version = "2.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, + { name = "h11" }, + { name = "truststore" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/83/a896fc59940fc5a6e2aff3a4be1d92fa890112936803b331cae75a993c34/httpcore2-2.10.0.tar.gz", hash = "sha256:13c0cc3d1919d4f28457f60cd2c2abe04113a8af184ccf1142811beba936f9dc", size = 67427, upload-time = "2026-08-09T09:11:32.123Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, + { url = "https://files.pythonhosted.org/packages/e5/4f/d149104195a35e2853a2fc203a8e3477747e58c80e17dda686dace174383/httpcore2-2.10.0-py3-none-any.whl", hash = "sha256:7df06cfb34070cae4f7c89be69dc1095eca138e9704ceffb98d25c1912ab6f01", size = 83000, upload-time = "2026-08-09T09:11:29.555Z" }, +] + +[[package]] +name = "httptools" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/e5/d471fcb0e14523fe1c3f4ba58ca52480e7bd70ad7109a3846bc75892f7fb/httptools-0.8.0.tar.gz", hash = "sha256:6b2a32f18d97e16e90827d7a819ffa8dbd8cc245fc4e1fa9d1095b54ef4bd999", size = 271342, upload-time = "2026-05-25T22:17:48.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/d4/13025f1a56e615dcb331e0bbe2d9a1143212b58c263385fc5d2e558f5bac/httptools-0.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:57278e6fa0424c42a8a3e454828ab4f0aff27b40cddf9679579b98c6dce6a376", size = 464676, upload-time = "2026-05-25T22:17:02.014Z" }, + { url = "https://files.pythonhosted.org/packages/bf/95/4c1c26c0b985f8a3331682d802598f14e32dc41bf7509266eb2c04ad4801/httptools-0.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bbb8caadb2b742d293169d2b458b5c001ef70e3158704aa3d3ef9597624c5d1d", size = 464235, upload-time = "2026-05-25T22:17:03.109Z" }, + { url = "https://files.pythonhosted.org/packages/a2/82/6735be2b0ca527718c431cdb8e5f70c3862c0844a687df0f572c51e11497/httptools-0.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:52dd695b865fe96d9d2b16b64a895f3f57bf3cb064e8383cd3b5713a069e8085", size = 449809, upload-time = "2026-05-25T22:17:04.443Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f9/5811c74f37a758c8a4aa3dc430375119d335947e883efc4664d8f3559a41/httptools-0.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:20b4aac66ff65f7db06a375808b78f42a94970aa22e826b3cb2b43eb09174124", size = 452174, upload-time = "2026-05-25T22:17:05.476Z" }, + { url = "https://files.pythonhosted.org/packages/e3/a6/febbb8b8db0f58b38e44ad6cb946e6a255ae49b55f2e8543408fb7501ccd/httptools-0.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b15fc622b0f869d19207c4089a501d9bcc63ca5e071ffdd2f03f922df882dcb2", size = 523851, upload-time = "2026-05-25T22:17:10.106Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e4/f90a0df0b83beff265b7e3b65f2a4cefd95792d4be0ac3e16049f2acd3c2/httptools-0.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:425f83884fd6343828d8c565f046cb72b6d19063f6924093e11bcd8e1548cd09", size = 518842, upload-time = "2026-05-25T22:17:11.218Z" }, + { url = "https://files.pythonhosted.org/packages/9e/2d/0c9ac76dd2c893841fbf6498d6acec4f2442e1b7067f6e3e316a80e494e8/httptools-0.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ef7c3c97f4311c7be57e2986629df89d49cb434dbff78eafcd48c2bff986b15a", size = 501238, upload-time = "2026-05-25T22:17:12.728Z" }, + { url = "https://files.pythonhosted.org/packages/ca/42/906adc91ae3a5fa9c59c0a2f21c139725bd7e5b41ae6acd485cd14123ebf/httptools-0.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a1afd7c9fbff0d9f5d489c4ce2768bd09c84a46ddefc7161e6aa82ae35c85745", size = 509567, upload-time = "2026-05-25T22:17:13.842Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1b/46f1cecf06b9bbde8e4b8c88034ac7908989e5ff7a3a388ef38392949c1f/httptools-0.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:eb3028cca2fc0a6d720e52ef61d8ebb62fcbfeb1de56874546d858d3f25a26b7", size = 486447, upload-time = "2026-05-25T22:17:18.564Z" }, + { url = "https://files.pythonhosted.org/packages/77/00/258bfc0837221f81d9725c45f9b948a6a6b2994a147a4fb66e85100c668f/httptools-0.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:88bdd940f2b5d487b4d032c6afa5489a7dc4694410d43de3c38c4fb3af0dc45d", size = 482448, upload-time = "2026-05-25T22:17:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/04/ab/d1cef3b5523f4d272a70f42a776c3169a2dddfe3a54de4b2ce4a36341528/httptools-0.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6a43c9dd399758ccc0531acb0a3c4a6c299ee893ee9400e9c893b7bdcfae0681", size = 464460, upload-time = "2026-05-25T22:17:20.882Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/5d1d072442277bb2b3434e0e60690b8e8c23840ef7de8b6ea54040a536d3/httptools-0.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0770728beb05094c809b98e814edff5fef69d26ad7d21185f2f6d5884a0ba683", size = 471312, upload-time = "2026-05-25T22:17:22.085Z" }, + { url = "https://files.pythonhosted.org/packages/ba/53/771bd891eb0f236f32145d6a1775777ec85745f3cc983a1f23d1a3b8ddfe/httptools-0.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c4a9f1707e4823d54dfec6c33fa3697d302aed536ed352a7ebb5a061ddb869d0", size = 481596, upload-time = "2026-05-25T22:17:26.186Z" }, + { url = "https://files.pythonhosted.org/packages/62/42/94e15bc68ce3d423243c45d7f1b0c7561f13844f97dc52ae23182fb65628/httptools-0.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d76ad7b951387e3632c8716a9bb03ac5b45c5f16119aa409db0459520887944e", size = 480865, upload-time = "2026-05-25T22:17:27.542Z" }, + { url = "https://files.pythonhosted.org/packages/1c/7c/fe2980fc03723272e30f135b62360b075f513dfe7cc73aef36c7f04012bd/httptools-0.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a3b7387147361c3fd47a0bde763c5c91b5b4cd4dc9989b8ece84ff436c99843b", size = 463189, upload-time = "2026-05-25T22:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/47fc5fff68acd1bfa20b4734059c9a06cadb88119dcd5258b5b0d21d91c8/httptools-0.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f256d6ce930c52ca1cb2a960b7da03548c454e7d28b06059ad41bfe789036ce0", size = 466610, upload-time = "2026-05-25T22:17:29.816Z" }, + { url = "https://files.pythonhosted.org/packages/fc/37/fccd705f795386bb05bf413012fecff2a33e5aa8c2f069096de3e9fd8702/httptools-0.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2a021c3a8e65cc125390d72f59b968afca3bdcaff25bd67965e0a055a14946ca", size = 558497, upload-time = "2026-05-25T22:17:34.732Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/f172e8003576de35f5ba77ff417cf0e34429d35dc014deef15afa337a72c/httptools-0.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48774d39cbb70e2b1f71f88852a3087ae1d3a1eb80482bb48c13067ab080c14f", size = 571585, upload-time = "2026-05-25T22:17:35.813Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b9/f5564760af99f3dbbf3f9104dc00e5da27e96cf433c6bdcf77617f70bf3f/httptools-0.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:88eead8ec8680a9f146c655bc88445a325bd7921cfd8194c7337e9467282427d", size = 543297, upload-time = "2026-05-25T22:17:37.08Z" }, + { url = "https://files.pythonhosted.org/packages/99/67/8d9f2c313618e161b82f3873188e7196126da1d6e29688df40eb3997c77a/httptools-0.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2c032fa028f46871ec7e1fc59fc15e8023eab3e6bbe6ece786a1611719a5d081", size = 539535, upload-time = "2026-05-25T22:17:38.032Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] [[package]] @@ -1273,25 +1942,40 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, ] +[[package]] +name = "httpx2" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpcore2" }, + { name = "idna" }, + { name = "truststore" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/3d/f9a8c07a3884f3e5b26205e8436a18b3af61c5d53192c3bea235574dbbec/httpx2-2.10.0.tar.gz", hash = "sha256:8741d7329fe2c7885fc9ceb61c8217acfb87a85f75723714b89ebf7ad7196338", size = 98749, upload-time = "2026-08-09T09:11:33.24Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/6d/a637d52449d98a6892d9a4dc0262587afdb6a66f201871842dce5a97b1c1/httpx2-2.10.0-py3-none-any.whl", hash = "sha256:5e3194a432701e1cc6f69a8b1b2fa199ef907013fede8d9a09a2c5b7b8141a18", size = 94355, upload-time = "2026-08-09T09:11:30.882Z" }, +] + [[package]] name = "huggingface-hub" -version = "1.4.1" +version = "1.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "click" }, { name = "filelock" }, { name = "fsspec" }, { name = "hf-xet", marker = "platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64'" }, { name = "httpx" }, { name = "packaging" }, { name = "pyyaml" }, - { name = "shellingham" }, { name = "tqdm" }, - { name = "typer-slim" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c4/fc/eb9bc06130e8bbda6a616e1b80a7aa127681c448d6b49806f61db2670b61/huggingface_hub-1.4.1.tar.gz", hash = "sha256:b41131ec35e631e7383ab26d6146b8d8972abc8b6309b963b306fbcca87f5ed5", size = 642156, upload-time = "2026-02-06T09:20:03.013Z" } +sdist = { url = "https://files.pythonhosted.org/packages/3e/9b/ddf3d02a8681f1b9ce52fda03d755dad6b74c4f8172304c4c8d2975450f9/huggingface_hub-1.27.0.tar.gz", hash = "sha256:c1fed40ea82a6b41b477f5243546549b792ae0a93abcea608cff66089bf8f8df", size = 942668, upload-time = "2026-08-07T12:48:05.161Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/ae/2f6d96b4e6c5478d87d606a1934b5d436c4a2bce6bb7c6fdece891c128e3/huggingface_hub-1.4.1-py3-none-any.whl", hash = "sha256:9931d075fb7a79af5abc487106414ec5fba2c0ae86104c0c62fd6cae38873d18", size = 553326, upload-time = "2026-02-06T09:20:00.728Z" }, + { url = "https://files.pythonhosted.org/packages/de/d8/95b735e183957c1f26d94c52977f09d466d55119cbbc1558ea4975e4c216/huggingface_hub-1.27.0-py3-none-any.whl", hash = "sha256:7df6827c2f956c60fbaa64646e979e566db76f619dd0a9729dfb8c5a3eb4f68d", size = 784926, upload-time = "2026-08-07T12:48:02.905Z" }, ] [[package]] @@ -1312,6 +1996,47 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] +[[package]] +name = "ijson" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/06/b31f040a8764336a11152e474a7abcb3782fedb0d1cdf78f442b82878c56/ijson-3.5.1.tar.gz", hash = "sha256:af40bd1a85f55db0b8b30715c858761306bd92d5590148636f75c3309e6e76bd", size = 69913, upload-time = "2026-07-06T17:37:42.923Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/b1/bc07831e646aebcc91a7bad9c5a0bf7c3f3395f0b10599e021667a3777f1/ijson-3.5.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e6cf9e49902f28af7a2e2f8b35c201195c0f0d5c170a5786e0c0a1b8492a4e37", size = 132095, upload-time = "2026-07-06T17:36:19.022Z" }, + { url = "https://files.pythonhosted.org/packages/1d/1f/b4547461d75db40744616e40c0a06cf2f46a14e60742f6d12510f4612985/ijson-3.5.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6ee1e6d59c800aa819952f6cb5ff08707ecd576b29cc9c3d00e33c2b371a92ce", size = 138790, upload-time = "2026-07-06T17:36:20.22Z" }, + { url = "https://files.pythonhosted.org/packages/a7/30/7ecba8377509eaea2666db5b39a1a99e23f5e3e1e7ee371ec366cbfc4f7c/ijson-3.5.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:affb85eb75fa03a21d1f790bbf26a0e66e5701672062a30dc5c3c6a29c5c0a63", size = 135233, upload-time = "2026-07-06T17:36:21.252Z" }, + { url = "https://files.pythonhosted.org/packages/38/36/0679010904b24398336b3099b09ccb1daa41c534e7cb0931e89d5fcdbee4/ijson-3.5.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3060b141ef758be3742315d44476109460c265b88247e3a4e479949f8b134eac", size = 138832, upload-time = "2026-07-06T17:36:22.323Z" }, + { url = "https://files.pythonhosted.org/packages/b0/90/a40f971e78191e423c7b3a23756f37c3a51c27aadd7769b3fb1816e0044d/ijson-3.5.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:ffba9bce60be21b496afc67a05ab8e3f431f87f0282fd6ce3c62004c951a1428", size = 133313, upload-time = "2026-07-06T17:36:23.405Z" }, + { url = "https://files.pythonhosted.org/packages/7b/d7/b012c347d3ab011c0c4f7988dc6e85b83eaab59df1aec089f5db0e7b29c5/ijson-3.5.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:170cc4c209f57decc9b7ee5fd340f2a1602d54020fa222846482ff1c99e88fdc", size = 135706, upload-time = "2026-07-06T17:36:24.464Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a5/9af7be670381ddac26dd55107ed0110b50f5161673b053311db67f510dcc/ijson-3.5.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ea4fd7bec203a600b1cc88a492dfe6b75ce4b1b87488a66adcd5406022213f64", size = 139092, upload-time = "2026-07-06T17:36:31.749Z" }, + { url = "https://files.pythonhosted.org/packages/41/fb/f9c1664d75467453e6bd4e5f9cd2211b730b09e049445ab64cbac68cc6a3/ijson-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:350caea815e53151994b597abc80cf669454276b5ac6aadcec69ef6d48f7e90b", size = 149921, upload-time = "2026-07-06T17:36:32.912Z" }, + { url = "https://files.pythonhosted.org/packages/43/80/d20b1c49c4aa7cc6644131e2e57192b45346ef4816566ed1cd9fd05bae38/ijson-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e4fcebfe1685bb7ba06a8255a5d428ea6b4b895d7acf979cb637d8bbc9db2f47", size = 149848, upload-time = "2026-07-06T17:36:34.032Z" }, + { url = "https://files.pythonhosted.org/packages/fd/fc/5baa710869f5ab939e6233583ced1546889b55c35f35b844c518ac10abc3/ijson-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d78f362f51c8691798758a9e6ac3c9d385ee1228cb82987c91562a2fae235cd3", size = 150810, upload-time = "2026-07-06T17:36:35.19Z" }, + { url = "https://files.pythonhosted.org/packages/54/16/a12b3d987a5c1677b04557c6f9b9feb7e04b7d4171e9a344856cb9136e9b/ijson-3.5.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0b184180d45f85fd4479659582749b109e49f4a29c21ac700ccc9c2280fe015e", size = 142989, upload-time = "2026-07-06T17:36:36.23Z" }, + { url = "https://files.pythonhosted.org/packages/ed/63/1026c535671fc334fc85aeb78f0945c825e7a338575edc753c0f455459ae/ijson-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e353891d33a2e6aa5caf72c2a5fbadd7a46f5f9b32dcfd0c84113b2444c255b8", size = 151702, upload-time = "2026-07-06T17:36:37.296Z" }, + { url = "https://files.pythonhosted.org/packages/f4/43/7bdca8f733c45ce97f61a64fadd3e51d255c4c9b467345cbf71ccc7bb368/ijson-3.5.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a19413a092d458a57aaa574fec08e265851d3b5c6e018377f426cd5e70b91280", size = 138889, upload-time = "2026-07-06T17:36:45.081Z" }, + { url = "https://files.pythonhosted.org/packages/e7/dc/e8a2e63700ab1d63aaf3fa38c454f8178eaa5b80a6d7c019d1d61b490a6c/ijson-3.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:65974568748678165d7e90e3e7ce2f7c233cfe4de6c37fbb0760941c97e14632", size = 149933, upload-time = "2026-07-06T17:36:46.312Z" }, + { url = "https://files.pythonhosted.org/packages/d9/56/640a4d980f7f2c11e399a7fd5ccb9e3d3c9e1dec3a1d5a10024570697c25/ijson-3.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bad5d55c99c89de8cd0a4cded51f86427ba3353c4dccca37ec2e32e06f26b437", size = 149857, upload-time = "2026-07-06T17:36:47.309Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a1/c953e22c83992b69ae538a83b3678d28768f1a48042fc7794733423a5ce7/ijson-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1a38d503ce343952e88edfd9a27296a4ec96af7073a9db58b3df6233367f75fc", size = 151141, upload-time = "2026-07-06T17:36:48.405Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ab/8fe5b7269b140e6e5f8837a33ce980fd9b67c70d0f8114289ed1cea4dace/ijson-3.5.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2f41982c73896acab4a2a14faa14e152e444bd69f37c3139204429fd3fe65a10", size = 143112, upload-time = "2026-07-06T17:36:50.353Z" }, + { url = "https://files.pythonhosted.org/packages/78/f3/23d1284edcde50ba337ddfba5b5d59f8273084d98b28af94715e73dd2b64/ijson-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3321fede2b638d400de0036889a3a25c3bb689feb8df45e70a393346aad6194f", size = 152184, upload-time = "2026-07-06T17:36:51.536Z" }, + { url = "https://files.pythonhosted.org/packages/3e/2b/5a55db881f1b043cd6d5716578937a60ac16348be1a3afbf846b21cf4b44/ijson-3.5.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:32f64051be2f990d8ae7b614b5abdf4a7bead510ce3666568d7403c6c46ce4d8", size = 140783, upload-time = "2026-07-06T17:36:58.984Z" }, + { url = "https://files.pythonhosted.org/packages/2e/61/f7783cc18672dc31544141139efd187fb34795d24e573fed6abea6b776c7/ijson-3.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cd0dfc5a788d0b0c2f1eab258b9dabdeefc631ca8ef87644a999f633b0b2555a", size = 149976, upload-time = "2026-07-06T17:37:00.235Z" }, + { url = "https://files.pythonhosted.org/packages/5f/d6/4182dd63b6b70eae4f5208c53558a050895a40734dff283463033c153742/ijson-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42bfda7858d99ee9777ec28cb6d347928249eefeb577f9b0a67503c18f7ebb6a", size = 149317, upload-time = "2026-07-06T17:37:01.476Z" }, + { url = "https://files.pythonhosted.org/packages/01/b1/a675e4a9b428a0ef556e7d718bf0e6885e3e5543042248a1a7030899a3d4/ijson-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c4b9a28e9719d1aebebe93ad8dc2ba87f4e2d9035043b196c1c07ef8530b44cc", size = 150555, upload-time = "2026-07-06T17:37:02.676Z" }, + { url = "https://files.pythonhosted.org/packages/b5/69/52686f56b44af63a93c3dc3f5bcfa07f87427d9aea4d2cbe3e1c94188c74/ijson-3.5.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:9a0b25c750a6bde14a0b31f1dcbfc86368e50767e3eaa73bb138e54128055edd", size = 144485, upload-time = "2026-07-06T17:37:03.779Z" }, + { url = "https://files.pythonhosted.org/packages/f0/46/10554e817dde56300a8414e52c0f5a44a29f3440327cd6d829ece57759b3/ijson-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:bd756f7b22df745ac14b7bc2ab9ed7c190a222e4c8e1bef26ef1162af8e54d0f", size = 151470, upload-time = "2026-07-06T17:37:04.901Z" }, + { url = "https://files.pythonhosted.org/packages/5b/01/95f3a7c27d25bb917954ef0c8e86d0e60f585b9db675cbd05d355f54cce8/ijson-3.5.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0ade373dd765b057b1dec05d7711bfeb5a36f1e825259466d9f545cfd8ef3ba3", size = 200568, upload-time = "2026-07-06T17:37:12.743Z" }, + { url = "https://files.pythonhosted.org/packages/77/61/c94ee4ea1f22318aab9a49b35d0ce8ac87dd24d508ea4c77dcbde362ba5e/ijson-3.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:882bc0bdd25d41eae90a15695cd50707edde0978b8b72a2532e30442dd8fd04c", size = 217956, upload-time = "2026-07-06T17:37:14.041Z" }, + { url = "https://files.pythonhosted.org/packages/1a/82/43e8d225aea5ee00eef7998c8ce41f344f7ba451329dfa9e92f4700813af/ijson-3.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:451901c36e12fa87cbb1cafe661bd25c08c6bd7900cc738279614f71cea07048", size = 208403, upload-time = "2026-07-06T17:37:15.201Z" }, + { url = "https://files.pythonhosted.org/packages/cf/6f/375f67fad76677aca9bc0817b2b18fdd231d309fe24e26b19a5556ef6cdd/ijson-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e3c5f660658f2ebfba5d4dfe4bafe8cd3a0defcda410ec08d2205fe08c398940", size = 211967, upload-time = "2026-07-06T17:37:16.484Z" }, + { url = "https://files.pythonhosted.org/packages/dc/53/4c754c3ba18ec70b7086b91a4abd368358fc47cc9b3871afd50deef4fea1/ijson-3.5.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:29eb8f0c77a296a10843a1714ad4a5d561e604cda3c88585e9012cf2c1729b0a", size = 201020, upload-time = "2026-07-06T17:37:18.017Z" }, + { url = "https://files.pythonhosted.org/packages/26/2d/3e7191b3222a31c378b827565b4fa64676a293441279f84db3d971720bf5/ijson-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:85997568d6b304cfa59d5c3f2b04f95b92e9a8c7f57d312343a7989cf8dfff85", size = 205584, upload-time = "2026-07-06T17:37:19.343Z" }, + { url = "https://files.pythonhosted.org/packages/9f/5b/553ea8f14dfc756d6b6c9be2e2231ab44877ce96408eb9da3bb3f11ddd13/ijson-3.5.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:0d7c5025a820f36f3e0e64f4b0232b338c690664c12b497e205cf64dcc64fc12", size = 71344, upload-time = "2026-07-06T17:37:38.997Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3e/0248fd00746731074ca01365a25d8aa3c4d54642c8a14490d94f7550bda9/ijson-3.5.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa7a2c94e43c02e0482088e6ff997e2bd7b9a76e6f1d0fd70891b4b5ff51318f", size = 71335, upload-time = "2026-07-06T17:37:39.965Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b9/1f1259546cc875adad240c468515f428d3a79b3def3ced17be3cdfe29146/ijson-3.5.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:69b5eef70240e9734c5a2fb5cc3742cae411fc833a66b9a50722b9eedb1e27de", size = 68728, upload-time = "2026-07-06T17:37:40.928Z" }, +] + [[package]] name = "iniconfig" version = "2.3.0" @@ -1321,6 +2046,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "interegular" +version = "0.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/9d/8b6dde58a028a3962ce17e84d5fe73758df61378e00ef8ac3d85da34b0ff/interegular-0.3.3.tar.gz", hash = "sha256:d9b697b21b34884711399ba0f0376914b81899ce670032486d0d048344a76600", size = 24705, upload-time = "2024-01-06T23:01:22.372Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/01/72d6472f80651673716d1deda2a5bbb633e563ecf94f4479da5519d69d25/interegular-0.3.3-py37-none-any.whl", hash = "sha256:b0c07007d48c89d6d19f7204972d369b2a77222722e126b6aa63aa721dc3b19c", size = 23635, upload-time = "2024-01-06T23:01:20.829Z" }, +] + [[package]] name = "ipykernel" version = "7.3.0" @@ -1431,6 +2165,73 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, ] +[[package]] +name = "jiter" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/89/d8d073f8aa2667e46c6c0873f86fe4a512bba4293cc730f626a076211a62/jiter-0.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64c0203212098470032aabcde9356fc168f377aade3e43def61dfe17e92f2037", size = 340939, upload-time = "2026-06-29T13:02:55.412Z" }, + { url = "https://files.pythonhosted.org/packages/87/c9/db4fda3ed73fb864139305e935e5b8b38a5a24692a5a9dd356c22f1b9c8d/jiter-0.16.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:12288303c9844e61e1651d02a9a6f6633e47d39f897d6991d1427161ce6b746e", size = 364932, upload-time = "2026-06-29T13:02:57.28Z" }, + { url = "https://files.pythonhosted.org/packages/a2/74/52b5e86241057f52ddd7c9a580f90effb51f9d06239f6fc612279b91a838/jiter-0.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5cf109d010b4b05a105afb3d43be36a21322d345ad3111e13d15f680afef0e5b", size = 461132, upload-time = "2026-06-29T13:02:58.994Z" }, + { url = "https://files.pythonhosted.org/packages/a9/87/544a700f7447c1f31c5d7833821a4daa5683165c2d5a094fbf5b5800c3dc/jiter-0.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:62c1b7fe1f77925acf5af68b6140b8810fa87dfd4dc0a9c8568ec2fa2a10429c", size = 374857, upload-time = "2026-06-29T13:03:00.455Z" }, + { url = "https://files.pythonhosted.org/packages/40/cd/0fcc3f7d39183674d5bfa9ec640faaeb506c60be7c8f94625dfba366e37c/jiter-0.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8597d23c87f59294f83bcb6229b9ed1fccee13dbba967b46930d2f1759466fee", size = 347053, upload-time = "2026-06-29T13:03:02.045Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ae/c7e64e7932ad597fa395b61440b249ada6366716e25c6e08dd2afbd021e6/jiter-0.16.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:3126a5dbad56401989ac769aca0cb56005bfb3e2366eea0ca99d1a91c3c1ee03", size = 356153, upload-time = "2026-06-29T13:03:03.706Z" }, + { url = "https://files.pythonhosted.org/packages/d4/1c/1c719044f14da814e1a060191ab19b96f3e99207bc5b4bfc6d6be34b3f80/jiter-0.16.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c4b4717bdb35ae456f831a6b08d01880fff399887a6bbc526a583a406e484eea", size = 393956, upload-time = "2026-06-29T13:03:05.165Z" }, + { url = "https://files.pythonhosted.org/packages/3b/dc/7b2f303a2847207e265503853a2d964a55354cffd62a5f2936c155486798/jiter-0.16.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:adff21bc78edfe086c15eb495b900306076de378dc2337c132401fc39bd79c91", size = 521081, upload-time = "2026-06-29T13:03:06.886Z" }, + { url = "https://files.pythonhosted.org/packages/c2/5f/501cf6e1e09caeb420195179ffc6f62aca603f1220ec53fd80d0d70b3e56/jiter-0.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dab907db06fc593645e73109acf4581ba5b548897d28b9348dc41ddc8343b2d3", size = 552085, upload-time = "2026-06-29T13:03:08.339Z" }, + { url = "https://files.pythonhosted.org/packages/88/6c/59bd309cab4460c54cf1079f3eb7fe7af6a4c895c5c957a53378693bad2b/jiter-0.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72", size = 335826, upload-time = "2026-06-29T13:03:17.11Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8c/f5ef7b65f0df47afa16596969defb281ebb86e96df346d62be6fd853d620/jiter-0.16.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e", size = 362573, upload-time = "2026-06-29T13:03:18.781Z" }, + { url = "https://files.pythonhosted.org/packages/2b/0b/ace4354da061ee38844a0c27dc2c21eecd27aea119e8da324bea987522d0/jiter-0.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290", size = 457979, upload-time = "2026-06-29T13:03:20.293Z" }, + { url = "https://files.pythonhosted.org/packages/55/40/c0253d3772eb9dcd8e6606ee9b2d53ec8e5b814589c47f140aa585f21eaa/jiter-0.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01", size = 372302, upload-time = "2026-06-29T13:03:21.739Z" }, + { url = "https://files.pythonhosted.org/packages/a8/d2/4839422241aa12860ce597b20068727094ba0bc480723c74924ca5bad483/jiter-0.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48", size = 343805, upload-time = "2026-06-29T13:03:23.384Z" }, + { url = "https://files.pythonhosted.org/packages/e2/59/e196888a05befdda7dbe299b722d56f2f6eec65402bc34c0a3306d595feb/jiter-0.16.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9", size = 351107, upload-time = "2026-06-29T13:03:24.815Z" }, + { url = "https://files.pythonhosted.org/packages/ec/74/4cd9e0fca65232136400354b630fbfcd2de634e22ccbb96567725981b548/jiter-0.16.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5", size = 388441, upload-time = "2026-06-29T13:03:26.266Z" }, + { url = "https://files.pythonhosted.org/packages/d9/8c/554691e48bc711299c0a293dd8a6179e24b2d66a54dc295421fcf64569c0/jiter-0.16.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730", size = 516354, upload-time = "2026-06-29T13:03:28.02Z" }, + { url = "https://files.pythonhosted.org/packages/a4/cb/01e9d69dc2cc6759d4f91e230b34489c4fdb2518992650633f9e20bece89/jiter-0.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f", size = 547880, upload-time = "2026-06-29T13:03:29.534Z" }, + { url = "https://files.pythonhosted.org/packages/96/3f/02fdfc6705cad96127d883af5c34e4867f554f29ec7705ec1a46156400a9/jiter-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536", size = 335453, upload-time = "2026-06-29T13:03:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a6/e4bda5920d4b0d7c5dfb7174ce4a6b2e4d3e11c9162c452ef0eab4cdbdbd/jiter-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450", size = 361625, upload-time = "2026-06-29T13:03:40.597Z" }, + { url = "https://files.pythonhosted.org/packages/b7/97/4e6b59b2c6e55cbb3e183595f81ad65dcfb21c915fee5e19e335df21bc55/jiter-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e", size = 456958, upload-time = "2026-06-29T13:03:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/15/e0/97e9557686d2f94f4b93786eccb7eed28e9228ad132ea8237f44727314a7/jiter-0.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8", size = 372017, upload-time = "2026-06-29T13:03:43.658Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/db768b6938e0df35c86beeba3dfbbb025c9ee5c19e1aa271f2396e50864d/jiter-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026", size = 343320, upload-time = "2026-06-29T13:03:45.226Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d6/5a59d938244a30735fe62d9433fd325f9021ea29d89780ea4596ea93bc89/jiter-0.16.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053", size = 350520, upload-time = "2026-06-29T13:03:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/c4a857f49c9af125f6bbcac7e3eee7f7978ed89682833062e2dbf62576b1/jiter-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e", size = 387550, upload-time = "2026-06-29T13:03:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/5fbc2f7d6b67b754caa61a993a2e626e815dec47ffc2f9e35f01adfebec7/jiter-0.16.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb", size = 515424, upload-time = "2026-06-29T13:03:49.881Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/284f0164b64a5fed915fea6ba7e9ba9b3d8d37c67d59cf2e3bb99d45cdfe/jiter-0.16.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84", size = 546981, upload-time = "2026-06-29T13:03:51.363Z" }, + { url = "https://files.pythonhosted.org/packages/11/fc/1f4fb7ebf9a724c7741994f4aae18fba1e2f3133df14521a79194952c34a/jiter-0.16.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883", size = 336998, upload-time = "2026-06-29T13:04:00.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/8d/72cadaac05ccfa7cc3a0a2232862e6c72443ca40cf300ba8b57f9f18b69b/jiter-0.16.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898", size = 362112, upload-time = "2026-06-29T13:04:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/58/4a/c4b0d5f651fda90a24ffce9f8d56cde462a2e09d31ae3de3c68cef34c04e/jiter-0.16.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5", size = 459807, upload-time = "2026-06-29T13:04:03.214Z" }, + { url = "https://files.pythonhosted.org/packages/80/58/ef77879ea9aa56b50824edc5a445e226422c7a8d211f3fd2a56bcb9493cf/jiter-0.16.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567", size = 373181, upload-time = "2026-06-29T13:04:04.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/2e/ffbc3f254e4d8a66da3062c624a7df4b7c2b2cf9e1fe43cf394b3e104041/jiter-0.16.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69", size = 344927, upload-time = "2026-06-29T13:04:06.067Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/0be5dc6d64a89f80aa8fec984f94dedb2973e251edcae55841d60786d578/jiter-0.16.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29", size = 352754, upload-time = "2026-06-29T13:04:07.477Z" }, + { url = "https://files.pythonhosted.org/packages/da/6e/7d31243b3b91cd261dd19e9d3557fc3251a80883d3d8049c86174e7ab7af/jiter-0.16.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93", size = 390553, upload-time = "2026-06-29T13:04:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/25/33/51ae371fde3c88897520f62b4d5f8b27ad7103e2bb10812ff52195609853/jiter-0.16.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a", size = 516900, upload-time = "2026-06-29T13:04:10.407Z" }, + { url = "https://files.pythonhosted.org/packages/a0/45/6449b3d123ea439ba79507c657288f461d55049e7bcbdc2cf8eb8210f491/jiter-0.16.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00", size = 548754, upload-time = "2026-06-29T13:04:12.046Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e7/fd2fb11ae3e2649333da3aa170d04d7b3000bbdc3b270f6513382fdf4e04/jiter-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe", size = 122381, upload-time = "2026-06-29T13:04:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/2d/95/49461034d5388196d3dabf98748935f017b7785d8f3f5349f834bcc4ed0d/jiter-0.16.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818", size = 340911, upload-time = "2026-06-29T13:04:21.257Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/a4369f2fb82cb3dda13b98622f31249b2e014b223fe64ee534413ad72294/jiter-0.16.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3", size = 361747, upload-time = "2026-06-29T13:04:22.677Z" }, + { url = "https://files.pythonhosted.org/packages/28/51/49b6ed456261646e1906016a6760367a28aacd3c24805e4e5fe64116c1db/jiter-0.16.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284", size = 460225, upload-time = "2026-06-29T13:04:24.441Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/5689aff4f66c5b60be63106e591dbfcba2190df97d2c9c7cf052361ddb98/jiter-0.16.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929", size = 373169, upload-time = "2026-06-29T13:04:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/a2/96/3ae1b85ee0d6d6cab254fb7f8da018272b932bbf2d69b07e98aa2a96c746/jiter-0.16.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620", size = 350332, upload-time = "2026-06-29T13:04:27.302Z" }, + { url = "https://files.pythonhosted.org/packages/15/32/c99d7bafd78986556c95bf60ce84c6cc98786eac56066c12d7f828bb6747/jiter-0.16.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af", size = 353377, upload-time = "2026-06-29T13:04:28.731Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/f99a8e571287c3dec766bcc18528bbe8e8fb5365522ab5e6d64c93e87066/jiter-0.16.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e", size = 387746, upload-time = "2026-06-29T13:04:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/c78a5b3f71040e34eb5917df26fb7ae9a2174cad1ccbf277512507c53a6e/jiter-0.16.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077", size = 517292, upload-time = "2026-06-29T13:04:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f7/095b38eda4c70d03651c403f29a5590f16d12ddc5d544aac9f9cddf72277/jiter-0.16.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734", size = 549259, upload-time = "2026-06-29T13:04:33.721Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ca/c366b1012da1d640de975d9683acd44e4d150d9068845d0ca2610435253f/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b7dc85ea77d4abbae8bad0d3538678aedee75bceec4e2f6c8dfb1c74772e5aa5", size = 342771, upload-time = "2026-06-29T13:05:03.55Z" }, + { url = "https://files.pythonhosted.org/packages/16/52/50cc4056fc1ae02e7154704e7ecc89df0afb8300222cfe8a52d3f67e4730/jiter-0.16.0-graalpy311-graalpy242_311_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17ca7fae79f6d99cd9a042b75f917eaada7b895cfc7dd2ee3a16089dcaec7a85", size = 346468, upload-time = "2026-06-29T13:05:05.452Z" }, + { url = "https://files.pythonhosted.org/packages/0a/db/bba1155f01a01c3c37a89425d571da751bbedf5c54247b831a04cb971798/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702", size = 339719, upload-time = "2026-06-29T13:05:10.41Z" }, + { url = "https://files.pythonhosted.org/packages/78/f7/18a1afcd64f35314b68c1f23afcd9994d0bc13e65cc77517afff4e83986d/jiter-0.16.0-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2", size = 343885, upload-time = "2026-06-29T13:05:12.087Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "json-repair" version = "0.57.1" @@ -1693,7 +2494,7 @@ dependencies = [ { name = "overrides", marker = "python_full_version < '3.12'" }, { name = "packaging" }, { name = "prometheus-client" }, - { name = "pywinpty", marker = "os_name == 'nt'" }, + { name = "pywinpty", marker = "os_name == 'nt' and sys_platform != 'darwin'" }, { name = "pyzmq" }, { name = "send2trash" }, { name = "terminado" }, @@ -1711,7 +2512,7 @@ name = "jupyter-server-terminals" version = "0.5.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywinpty", marker = "os_name == 'nt'" }, + { name = "pywinpty", marker = "os_name == 'nt' and sys_platform != 'darwin'" }, { name = "terminado" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f4/a7/bcd0a9b0cbba88986fe944aaaf91bfda603e5a50bda8ed15123f381a3b2f/jupyter_server_terminals-0.5.4.tar.gz", hash = "sha256:bbda128ed41d0be9020349f9f1f2a4ab9952a73ed5f5ac9f1419794761fb87f5", size = 31770, upload-time = "2026-01-14T16:53:20.213Z" } @@ -1797,11 +2598,66 @@ wheels = [ [[package]] name = "lark" -version = "1.3.1" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/60/bc7622aefb2aee1c0b4ba23c1446d3e30225c8770b38d7aedbfb65ca9d5a/lark-1.2.2.tar.gz", hash = "sha256:ca807d0162cd16cef15a8feecb862d7319e7a09bdb13aef927968e45040fed80", size = 252132, upload-time = "2024-08-13T19:49:00.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2d/00/d90b10b962b4277f5e64a78b6609968859ff86889f5b898c1a778c06ec00/lark-1.2.2-py3-none-any.whl", hash = "sha256:c2276486b02f0f1b90be155f2c8ba4a8e194d42775786db622faccd652d8e80c", size = 111036, upload-time = "2024-08-13T19:48:58.603Z" }, +] + +[[package]] +name = "llguidance" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/48/3f7a9d3ff1b36bba92b5107a3a21286821227afe9ea464736133994d61fb/llguidance-1.3.0.tar.gz", hash = "sha256:861249afd51dc325646834462ea827e57a5c2b2042e108e6aae7059fdad9104d", size = 1070460, upload-time = "2025-10-20T19:58:44.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/11/44389d3d1526d7a5c38ffd587a5ebc61d7bee443ac1dea95f2089ad58f5f/llguidance-1.3.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f6caca5d78db7f76e1fbb0fff8607b861c32d47fa3d5dee2fc49de27ee269df", size = 2835242, upload-time = "2025-10-20T19:58:34.518Z" }, + { url = "https://files.pythonhosted.org/packages/83/a8/1ff2bedb8f9acb46a2d2d603415d272bb622c142ea86f5b95445cc6e366c/llguidance-1.3.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc17e9dd602c3879bf91664a64bf72f54c74dbfbeb24ccfab6a5fe435b12f7aa", size = 3033133, upload-time = "2025-10-20T19:58:38.721Z" }, +] + +[[package]] +name = "llvmlite" +version = "0.47.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/88/a8952b6d5c21e74cbf158515b779666f692846502623e9e3c39d8e8ba25f/llvmlite-0.47.0.tar.gz", hash = "sha256:62031ce968ec74e95092184d4b0e857e444f8fdff0b8f9213707699570c33ccc", size = 193614, upload-time = "2026-03-31T18:29:53.497Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/27/5799b020e4cdfb25a7c951c06a96397c135efcdc21b78d853bbd9c814c7d/llvmlite-0.47.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ca14f02e29134e837982497959a8e2193d6035235de1cb41a9cb2bd6da4eedbb", size = 56275177, upload-time = "2026-03-31T18:28:31.01Z" }, + { url = "https://files.pythonhosted.org/packages/7e/51/48a53fedf01cb1f3f43ef200be17ebf83c8d9a04018d3783c1a226c342c2/llvmlite-0.47.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12a69d4bb05f402f30477e21eeabe81911e7c251cecb192bed82cd83c9db10d8", size = 55128631, upload-time = "2026-03-31T18:28:36.046Z" }, + { url = "https://files.pythonhosted.org/packages/e6/4b/e3f2cd17822cf772a4a51a0a8080b0032e6d37b2dbe8cfb724eac4e31c52/llvmlite-0.47.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5853bf26160857c0c2573415ff4efe01c4c651e59e2c55c2a088740acfee51cd", size = 56275178, upload-time = "2026-03-31T18:28:48.342Z" }, + { url = "https://files.pythonhosted.org/packages/b6/55/a3b4a543185305a9bdf3d9759d53646ed96e55e7dfd43f53e7a421b8fbae/llvmlite-0.47.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:003bcf7fa579e14db59c1a1e113f93ab8a06b56a4be31c7f08264d1d4072d077", size = 55128632, upload-time = "2026-03-31T18:28:52.901Z" }, + { url = "https://files.pythonhosted.org/packages/31/b8/69f5565f1a280d032525878a86511eebed0645818492feeb169dfb20ae8e/llvmlite-0.47.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2699a74321189e812d476a43d6d7f652f51811e7b5aad9d9bba842a1c7927acb", size = 56275178, upload-time = "2026-03-31T18:29:05.748Z" }, + { url = "https://files.pythonhosted.org/packages/d6/da/b32cafcb926fb0ce2aa25553bf32cb8764af31438f40e2481df08884c947/llvmlite-0.47.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c6951e2b29930227963e53ee152441f0e14be92e9d4231852102d986c761e40", size = 55128632, upload-time = "2026-03-31T18:29:11.235Z" }, + { url = "https://files.pythonhosted.org/packages/64/1d/a760e993e0c0ba6db38d46b9f48f6c7dceb8ac838824997fb9e25f97bc04/llvmlite-0.47.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ddbccff2aeaff8670368340a158abefc032fe9b3ccf7d9c496639263d00151aa", size = 56275176, upload-time = "2026-03-31T18:29:24.149Z" }, + { url = "https://files.pythonhosted.org/packages/84/3b/e679bc3b29127182a7f4aa2d2e9e5bea42adb93fb840484147d59c236299/llvmlite-0.47.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4a7b778a2e144fc64468fb9bf509ac1226c9813a00b4d7afea5d988c4e22fca", size = 55128631, upload-time = "2026-03-31T18:29:29.536Z" }, + { url = "https://files.pythonhosted.org/packages/11/03/16090dd6f74ba2b8b922276047f15962fbeea0a75d5601607edb301ba945/llvmlite-0.47.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa1cbd800edd3b20bc141521f7fd45a6185a5b84109aa6855134e81397ffe72b", size = 56275178, upload-time = "2026-03-31T18:29:42.58Z" }, + { url = "https://files.pythonhosted.org/packages/f5/cb/0abf1dd4c5286a95ffe0c1d8c67aec06b515894a0dd2ac97f5e27b82ab0b/llvmlite-0.47.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6725179b89f03b17dabe236ff3422cb8291b4c1bf40af152826dfd34e350ae8", size = 55128632, upload-time = "2026-03-31T18:29:46.939Z" }, +] + +[[package]] +name = "lm-format-enforcer" +version = "0.11.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/34/28fff3ab31ccff1fd4f6c7c7b0ceb2b6968d8ea4950663eadcb5720591a0/lark-1.3.1.tar.gz", hash = "sha256:b426a7a6d6d53189d318f2b6236ab5d6429eaf09259f1ca33eb716eed10d2905", size = 382732, upload-time = "2025-10-27T18:25:56.653Z" } +dependencies = [ + { name = "interegular" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/84/d5/41cd417ba7dfdbbcfe46cebf81fb3dfd7c591b89897560ad05bb410a465d/lm_format_enforcer-0.11.3.tar.gz", hash = "sha256:e68081c108719cce284a9bcc889709b26ffb085a1945b5eba3a12cfa96d528da", size = 40258, upload-time = "2025-08-24T19:37:47.527Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/ef/11292bb0b85cf4c93447cab5a29f64576ed14d3ab4280e35ddd23486594a/lm_format_enforcer-0.11.3-py3-none-any.whl", hash = "sha256:cf586350875def1ae7a8fba84fcbbfc8371424b6c9d05c1fcba70aa233fbf06f", size = 45418, upload-time = "2025-08-24T19:37:46.325Z" }, +] + +[[package]] +name = "loguru" +version = "0.7.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "python_version < '0'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3a/05/a1dae3dffd1116099471c643b8924f5aa6524411dc6c63fdae648c4f1aca/loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6", size = 63559, upload-time = "2024-12-06T11:20:56.608Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, + { url = "https://files.pythonhosted.org/packages/0c/29/0348de65b8cc732daa3e33e67806420b2ae89bdce2b04af740289c5c6c8c/loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c", size = 61595, upload-time = "2024-12-06T11:20:54.538Z" }, ] [[package]] @@ -2094,6 +2950,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/f7/10f5e101db25741b91e4f4792c5d97b4fa834ead5cf509ae91097d939424/mike-2.1.4-py3-none-any.whl", hash = "sha256:39933e992e155dd70f2297e749a0ed78d8fd7942bc33a3666195d177758a280e", size = 33820, upload-time = "2026-03-08T02:46:28.149Z" }, ] +[[package]] +name = "mistral-common" +version = "1.11.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "pillow" }, + { name = "pydantic" }, + { name = "pydantic-extra-types", extra = ["pycountry"] }, + { name = "requests" }, + { name = "tiktoken" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/d0/61b2c24be62a8e2f0e46a1c16de23de386c8644408da249bc66768a6681b/mistral_common-1.11.7.tar.gz", hash = "sha256:d3b79583595cf6d96a2ab33e42cb8449768383147b8c56cac5a4f193be19d20d", size = 6387178, upload-time = "2026-07-23T09:21:17.206Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a4/bc2850eb33cc2d633a21f51530756350dca325ee69b07c9202552f2bbadb/mistral_common-1.11.7-py3-none-any.whl", hash = "sha256:a9511b88eacacbe7dacddd9d3498c1739f56847b7fdddbd5a22e7844fd9def95", size = 6553583, upload-time = "2026-07-23T09:21:19.818Z" }, +] + +[package.optional-dependencies] +image = [ + { name = "opencv-python-headless" }, +] + [[package]] name = "mistune" version = "3.2.0" @@ -2265,6 +3146,85 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/28/79f0f8de97cce916d5ae88a7bee1ad724855e83e6019c0b4d5b3fabc80f3/mkdocstrings_python-2.0.3-py3-none-any.whl", hash = "sha256:0b83513478bdfd803ff05aa43e9b1fca9dd22bcd9471f09ca6257f009bc5ee12", size = 104779, upload-time = "2026-02-20T10:38:34.517Z" }, ] +[[package]] +name = "ml-dtypes" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/c27b42ed9b1c7d13d9ba8b6905dece787d6259152f2309338aed29b2447b/ml_dtypes-0.5.4.tar.gz", hash = "sha256:8ab06a50fb9bf9666dd0fe5dfb4676fa2b0ac0f31ecff72a6c3af8e22c063453", size = 692314, upload-time = "2025-11-17T22:32:31.031Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/cf/912146dfd4b5c0eea956836c01dcd2fce6c9c844b2691f5152aca196ce4f/ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc11d7e8c44a65115d05e2ab9989d1e045125d7be8e05a071a48bc76eb6d6040", size = 5056165, upload-time = "2025-11-17T22:31:41.071Z" }, + { url = "https://files.pythonhosted.org/packages/a9/80/19189ea605017473660e43762dc853d2797984b3c7bf30ce656099add30c/ml_dtypes-0.5.4-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:19b9a53598f21e453ea2fbda8aa783c20faff8e1eeb0d7ab899309a0053f1483", size = 5034975, upload-time = "2025-11-17T22:31:42.758Z" }, + { url = "https://files.pythonhosted.org/packages/54/0f/428ef6881782e5ebb7eca459689448c0394fa0a80bea3aa9262cba5445ea/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a7f7c643e8b1320fd958bf098aa7ecf70623a42ec5154e3be3be673f4c34d900", size = 5028464, upload-time = "2025-11-17T22:31:50.135Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cb/28ce52eb94390dda42599c98ea0204d74799e4d8047a0eb559b6fd648056/ml_dtypes-0.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9ad459e99793fa6e13bd5b7e6792c8f9190b4e5a1b45c63aba14a4d0a7f1d5ff", size = 5009002, upload-time = "2025-11-17T22:31:52.001Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b7/dff378afc2b0d5a7d6cd9d3209b60474d9819d1189d347521e1688a60a53/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b", size = 5036993, upload-time = "2025-11-17T22:31:58.497Z" }, + { url = "https://files.pythonhosted.org/packages/eb/33/40cd74219417e78b97c47802037cf2d87b91973e18bb968a7da48a96ea44/ml_dtypes-0.5.4-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:533ce891ba774eabf607172254f2e7260ba5f57bdd64030c9a4fcfbd99815d0d", size = 5010956, upload-time = "2025-11-17T22:31:59.931Z" }, + { url = "https://files.pythonhosted.org/packages/74/f5/667060b0aed1aa63166b22897fdf16dca9eb704e6b4bbf86848d5a181aa7/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a0df4223b514d799b8a1629c65ddc351b3efa833ccf7f8ea0cf654a61d1e35d", size = 5354111, upload-time = "2025-11-17T22:32:05.546Z" }, + { url = "https://files.pythonhosted.org/packages/40/49/0f8c498a28c0efa5f5c95a9e374c83ec1385ca41d0e85e7cf40e5d519a21/ml_dtypes-0.5.4-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:531eff30e4d368cb6255bc2328d070e35836aa4f282a0fb5f3a0cd7260257298", size = 5366453, upload-time = "2025-11-17T22:32:07.115Z" }, + { url = "https://files.pythonhosted.org/packages/04/f9/067b84365c7e83bda15bba2b06c6ca250ce27b20630b1128c435fb7a09aa/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:805cef3a38f4eafae3a5bf9ebdcdb741d0bcfd9e1bd90eb54abd24f928cd2465", size = 5036145, upload-time = "2025-11-17T22:32:12.783Z" }, + { url = "https://files.pythonhosted.org/packages/c6/bb/82c7dcf38070b46172a517e2334e665c5bf374a262f99a283ea454bece7c/ml_dtypes-0.5.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:14a4fd3228af936461db66faccef6e4f41c1d82fcc30e9f8d58a08916b1d811f", size = 5010230, upload-time = "2025-11-17T22:32:14.38Z" }, + { url = "https://files.pythonhosted.org/packages/5a/e7/85cb99fe80a7a5513253ec7faa88a65306be071163485e9a626fce1b6e84/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2314892cdc3fcf05e373d76d72aaa15fda9fb98625effa73c1d646f331fcecb7", size = 5355358, upload-time = "2025-11-17T22:32:19.7Z" }, + { url = "https://files.pythonhosted.org/packages/79/2b/a826ba18d2179a56e144aef69e57fb2ab7c464ef0b2111940ee8a3a223a2/ml_dtypes-0.5.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d2ffd05a2575b1519dc928c0b93c06339eb67173ff53acb00724502cda231cf", size = 5366332, upload-time = "2025-11-17T22:32:21.193Z" }, +] + +[[package]] +name = "model-hosting-container-standards" +version = "0.1.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastapi" }, + { name = "httpx" }, + { name = "jmespath" }, + { name = "pydantic" }, + { name = "setuptools" }, + { name = "starlette" }, + { name = "supervisor" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/5f/bc0d0fce1bd0a35378696aa13b21feffa18d9cda837f4e1be124e45ee090/model_hosting_container_standards-0.1.16.tar.gz", hash = "sha256:d34589633900e53c3ee5f7c78280a7cf7e4f6532c35e763341a262fc85cbe84a", size = 94130, upload-time = "2026-06-15T21:29:34.771Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/ef/6eabeb251d2a0598cb5f9a274159e05ae07a1e3fe6a1473bf6035793252a/model_hosting_container_standards-0.1.16-py3-none-any.whl", hash = "sha256:47f4f65713120bc3a69feb022981a38db9e557aedf88dbd72077f20588caa12b", size = 125666, upload-time = "2026-06-15T21:29:33.415Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "msgspec" +version = "0.21.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/60/f79b9b013a16fa3a58350c9295ddc6789f2e335f36ea61ed10a21b215364/msgspec-0.21.1.tar.gz", hash = "sha256:2313508e394b0d208f8f56892ca9b2799e2561329de9763b19619595a6c0f72c", size = 319193, upload-time = "2026-04-12T21:44:50.394Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/54/d24ddeaa65b5278c9e67f48ce3c17a9831e8f3722f3c8322ee120aca22ef/msgspec-0.21.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d3124010b3815451494c85ff345e693cb9fe5889cfcbbef39ed8622e0e72319c", size = 215158, upload-time = "2026-04-12T21:43:58.442Z" }, + { url = "https://files.pythonhosted.org/packages/9f/75/bb79c8b89a93ae23cd33c0d802373f16feaf9633f05d8af77091350dda0a/msgspec-0.21.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6badc03b9725352219cca017bfe71c61f2fbd0fb5982b410ac17c97c213deb30", size = 219856, upload-time = "2026-04-12T21:44:00.015Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/c5ca26b46f0ebbd3a6683695ef89396712cb9e4199fd1f0bc1dd968216b1/msgspec-0.21.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:5d2d4116ebe3035a78d9ec76e99a9d64e5fa6d44fe61a9c5de7fd1acf54bcc69", size = 220314, upload-time = "2026-04-12T21:44:01.548Z" }, + { url = "https://files.pythonhosted.org/packages/c8/31/645a351c4285dce40ed6755c3dcc0aa648e26dacb20a98018fe2cce5e87b/msgspec-0.21.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0d1009f6715f5bff3b54d4ff5c7428ad96197e0534e1645b8e9b955890c84664", size = 223215, upload-time = "2026-04-12T21:44:02.884Z" }, + { url = "https://files.pythonhosted.org/packages/8a/37/655101799590bcc5fddb2bd3fe0e6194e816c2d1da7c361725f5eb89a910/msgspec-0.21.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:846758412e9518252b2ac9bffd6f0e54d9ff614f5f9488df7749f81ff5c80920", size = 218871, upload-time = "2026-04-12T21:44:09.917Z" }, + { url = "https://files.pythonhosted.org/packages/b5/d1/d4cd9fe89c7d400d7a18f86ccc94daa3f0927f53558846fcb60791dce5d6/msgspec-0.21.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21995e74b5c598c2e004110ad66ec7f1b8c20bf2bcf3b2de8fd9a3094422d3ff", size = 225025, upload-time = "2026-04-12T21:44:11.191Z" }, + { url = "https://files.pythonhosted.org/packages/24/bf/e20549e602b9edccadeeff98760345a416f9cce846a657e8b18e3396b212/msgspec-0.21.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6129f0cca52992e898fd5344187f7c8127b63d810b2fd73e36fca73b4c6475ee", size = 222672, upload-time = "2026-04-12T21:44:12.481Z" }, + { url = "https://files.pythonhosted.org/packages/b4/68/04d7a8f0f786545cf9b8c280c57aa6befb5977af6e884b8b54191cbe44b3/msgspec-0.21.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ef3ec2296248d1f8b9231acb051b6d471dfde8f21819e86c9adaaa9f42918521", size = 227303, upload-time = "2026-04-12T21:44:13.709Z" }, + { url = "https://files.pythonhosted.org/packages/ca/d9/9e9d7d7e5061b47540d03d640fab9b3965ba7ae49c1b2154861c8f007518/msgspec-0.21.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48943e278b3854c2f89f955ddc6f9f430d3f0784b16e47d10604ee0463cd21f5", size = 218880, upload-time = "2026-04-12T21:44:20.028Z" }, + { url = "https://files.pythonhosted.org/packages/74/66/2bb344f34abb4b57e60c7c9c761994e0417b9718ec1460bf00c296f2a7ea/msgspec-0.21.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a9aa659ebb0101b1cbc31461212b87e341d961f0ab0772aaf068a99e001ec4aa", size = 225050, upload-time = "2026-04-12T21:44:21.577Z" }, + { url = "https://files.pythonhosted.org/packages/1a/84/7c1e412f76092277bf760cef12b7979d03314d259ab5b5cafde5d0c1722d/msgspec-0.21.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7b27d1a8ead2b6f5b0c4f2d07b8be1ccfcc041c8a0e704781edebe3ae13c484", size = 222713, upload-time = "2026-04-12T21:44:22.83Z" }, + { url = "https://files.pythonhosted.org/packages/4e/27/0bba04b2b4ef05f3d068429410bc71d2cea925f1596a8f41152cccd5edb8/msgspec-0.21.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:38fe93e86b61328fe544cb7fd871fad5a27c8734bfda90f65e5dbe288ae50f61", size = 227259, upload-time = "2026-04-12T21:44:24.11Z" }, + { url = "https://files.pythonhosted.org/packages/85/7d/1e29a319d678d6cb962ae5bdf32a6858ebdf38f73bc654c0e9c742a0c2c8/msgspec-0.21.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:68604db36b3b4dd9bf160e436e12798a4738848144cea1aca1cb984011eb160f", size = 219866, upload-time = "2026-04-12T21:44:31.104Z" }, + { url = "https://files.pythonhosted.org/packages/25/1f/cca084ca2572810fff12ea9dbdcbe39eac048f40daf4a9077b49fcbe8cee/msgspec-0.21.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d6b9dc50948eaf65df54d2fd0ff66e6d8c32f116037209ee861810eb9b676cb", size = 224993, upload-time = "2026-04-12T21:44:32.649Z" }, + { url = "https://files.pythonhosted.org/packages/71/94/d2120fc9d419a89a3a7c13e5b7078798c4b392a96a02a6e2b3ce43a8766c/msgspec-0.21.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:52c5e21930942302394429c5a582ce7e6b62c7f983b3760834c2ce107e0dd6df", size = 223535, upload-time = "2026-04-12T21:44:33.839Z" }, + { url = "https://files.pythonhosted.org/packages/75/17/42418b66a3ad972a89bab73dd78b79cc6282bb488a25e73c853cee7443b9/msgspec-0.21.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:abbb39d65681fa24ed394e01af3d59d869068324f900c61d06062b7fb9980f2f", size = 227222, upload-time = "2026-04-12T21:44:35.093Z" }, + { url = "https://files.pythonhosted.org/packages/a7/08/673a7bb05e5702dc787ddd3011195b509f9867927970da59052211929987/msgspec-0.21.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f60800e6299b798142dc40b0644da77ceac5ea0568be58228417eae14135c847", size = 226281, upload-time = "2026-04-12T21:44:42.181Z" }, + { url = "https://files.pythonhosted.org/packages/7d/45/86508cf57283e9070b3c447e3ab25b792a7a0855a3ea4e0c6d111ac34c97/msgspec-0.21.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f8e9dfcd98419cf7568808470c4317a3fb30bef0e3715b568730a2b272a20d7", size = 229863, upload-time = "2026-04-12T21:44:43.442Z" }, + { url = "https://files.pythonhosted.org/packages/2c/62/e7c9367cd08d590559faacd711edbae36840342843e669440363f33c7d36/msgspec-0.21.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:92d89dfad13bd1ea640dc3e37e724ed380da1030b272bdf5ecafb983c3ad7c75", size = 230445, upload-time = "2026-04-12T21:44:44.806Z" }, + { url = "https://files.pythonhosted.org/packages/42/b4/c0f54632103846b658a10930025f4de41c8724b5e4805a5f3b395586cb7e/msgspec-0.21.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0d03867786e5d7ba25d666df4b11320c27170f4aeafcb8e3a8b0a50a4fb742ca", size = 231822, upload-time = "2026-04-12T21:44:46.343Z" }, +] + [[package]] name = "multidict" version = "6.7.1" @@ -2486,6 +3446,9 @@ docs = [ { name = "mkdocs-material" }, { name = "mkdocstrings", extra = ["python"] }, ] +local-models = [ + { name = "vllm", marker = "sys_platform == 'linux'" }, +] measurement = [ { name = "wandb", extra = ["workspaces"] }, ] @@ -2528,6 +3491,7 @@ docs = [ { name = "mkdocs-material" }, { name = "mkdocstrings", extras = ["python"] }, ] +local-models = [{ name = "vllm", marker = "sys_platform == 'linux'", specifier = "==0.20.0" }] measurement = [{ name = "wandb", extras = ["workspaces"], specifier = ">=0.19,<1" }] notebooks = [ { name = "datasets", specifier = ">=4.0.0,<6" }, @@ -2555,6 +3519,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c9/b2622292ea83fbb4ec318f5b9ab867d0a28ab43c5717bb85b0a5f6b3b0a4/networkx-3.6.1-py3-none-any.whl", hash = "sha256:d47fbf302e7d9cbbb9e2555a0d267983d2aa476bac30e90dfbe5669bd57f3762", size = 2068504, upload-time = "2025-12-08T17:02:38.159Z" }, ] +[[package]] +name = "ninja" +version = "1.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/73/79a0b22fc731989c708068427579e840a6cf4e937fe7ae5c5d0b7356ac22/ninja-1.13.0.tar.gz", hash = "sha256:4a40ce995ded54d9dc24f8ea37ff3bf62ad192b547f6c7126e7e25045e76f978", size = 242558, upload-time = "2025-08-11T15:10:19.421Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/de/6e1cd6b84b412ac1ef327b76f0641aeb5dcc01e9d3f9eee0286d0c34fd93/ninja-1.13.0-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3d00c692fb717fd511abeb44b8c5d00340c36938c12d6538ba989fe764e79630", size = 177467, upload-time = "2025-08-11T15:09:52.767Z" }, + { url = "https://files.pythonhosted.org/packages/c8/83/49320fb6e58ae3c079381e333575fdbcf1cca3506ee160a2dcce775046fa/ninja-1.13.0-py3-none-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:be7f478ff9f96a128b599a964fc60a6a87b9fa332ee1bd44fa243ac88d50291c", size = 187834, upload-time = "2025-08-11T15:09:54.115Z" }, + { url = "https://files.pythonhosted.org/packages/56/c7/ba22748fb59f7f896b609cd3e568d28a0a367a6d953c24c461fe04fc4433/ninja-1.13.0-py3-none-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:60056592cf495e9a6a4bea3cd178903056ecb0943e4de45a2ea825edb6dc8d3e", size = 202736, upload-time = "2025-08-11T15:09:55.745Z" }, + { url = "https://files.pythonhosted.org/packages/79/22/d1de07632b78ac8e6b785f41fa9aad7a978ec8c0a1bf15772def36d77aac/ninja-1.13.0-py3-none-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1c97223cdda0417f414bf864cfb73b72d8777e57ebb279c5f6de368de0062988", size = 179034, upload-time = "2025-08-11T15:09:57.394Z" }, + { url = "https://files.pythonhosted.org/packages/ed/de/0e6edf44d6a04dabd0318a519125ed0415ce437ad5a1ec9b9be03d9048cf/ninja-1.13.0-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fb46acf6b93b8dd0322adc3a4945452a4e774b75b91293bafcc7b7f8e6517dfa", size = 180716, upload-time = "2025-08-11T15:09:58.696Z" }, + { url = "https://files.pythonhosted.org/packages/54/28/938b562f9057aaa4d6bfbeaa05e81899a47aebb3ba6751e36c027a7f5ff7/ninja-1.13.0-py3-none-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4be9c1b082d244b1ad7ef41eb8ab088aae8c109a9f3f0b3e56a252d3e00f42c1", size = 146843, upload-time = "2025-08-11T15:10:00.046Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fb/d06a3838de4f8ab866e44ee52a797b5491df823901c54943b2adb0389fbb/ninja-1.13.0-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6739d3352073341ad284246f81339a384eec091d9851a886dfa5b00a6d48b3e2", size = 154402, upload-time = "2025-08-11T15:10:01.657Z" }, + { url = "https://files.pythonhosted.org/packages/31/bf/0d7808af695ceddc763cf251b84a9892cd7f51622dc8b4c89d5012779f06/ninja-1.13.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:11be2d22027bde06f14c343f01d31446747dbb51e72d00decca2eb99be911e2f", size = 552388, upload-time = "2025-08-11T15:10:03.349Z" }, + { url = "https://files.pythonhosted.org/packages/9d/70/c99d0c2c809f992752453cce312848abb3b1607e56d4cd1b6cded317351a/ninja-1.13.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:aa45b4037b313c2f698bc13306239b8b93b4680eb47e287773156ac9e9304714", size = 472501, upload-time = "2025-08-11T15:10:04.735Z" }, + { url = "https://files.pythonhosted.org/packages/9f/43/c217b1153f0e499652f5e0766da8523ce3480f0a951039c7af115e224d55/ninja-1.13.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5f8e1e8a1a30835eeb51db05cf5a67151ad37542f5a4af2a438e9490915e5b72", size = 638280, upload-time = "2025-08-11T15:10:06.512Z" }, + { url = "https://files.pythonhosted.org/packages/8c/45/9151bba2c8d0ae2b6260f71696330590de5850e5574b7b5694dce6023e20/ninja-1.13.0-py3-none-musllinux_1_2_ppc64le.whl", hash = "sha256:3d7d7779d12cb20c6d054c61b702139fd23a7a964ec8f2c823f1ab1b084150db", size = 642420, upload-time = "2025-08-11T15:10:08.35Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/95752eb635bb8ad27d101d71bef15bc63049de23f299e312878fc21cb2da/ninja-1.13.0-py3-none-musllinux_1_2_riscv64.whl", hash = "sha256:d741a5e6754e0bda767e3274a0f0deeef4807f1fec6c0d7921a0244018926ae5", size = 585106, upload-time = "2025-08-11T15:10:09.818Z" }, + { url = "https://files.pythonhosted.org/packages/c1/31/aa56a1a286703800c0cbe39fb4e82811c277772dc8cd084f442dd8e2938a/ninja-1.13.0-py3-none-musllinux_1_2_s390x.whl", hash = "sha256:e8bad11f8a00b64137e9b315b137d8bb6cbf3086fbdc43bf1f90fd33324d2e96", size = 707138, upload-time = "2025-08-11T15:10:11.366Z" }, + { url = "https://files.pythonhosted.org/packages/34/6f/5f5a54a1041af945130abdb2b8529cbef0cdcbbf9bcf3f4195378319d29a/ninja-1.13.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b4f2a072db3c0f944c32793e91532d8948d20d9ab83da9c0c7c15b5768072200", size = 581758, upload-time = "2025-08-11T15:10:13.295Z" }, +] + [[package]] name = "nodeenv" version = "1.10.0" @@ -2592,10 +3578,126 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef", size = 13307, upload-time = "2024-02-14T23:35:16.286Z" }, ] +[[package]] +name = "numba" +version = "0.65.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llvmlite" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/61/7299643b9c18d669e04be7c5bcb64d985070d07553274817b45b049e7bfe/numba-0.65.0.tar.gz", hash = "sha256:edad0d9f6682e93624c00125a471ae4df186175d71fd604c983c377cdc03e68b", size = 2764131, upload-time = "2026-04-01T03:52:01.946Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/a7/11e2b24251d57cf41fc9ad83f378d890d61a890e3f8eb6338b39833f67a4/numba-0.65.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:032b0b8e879512cd424d79eed6d772a1399c6387ded184c2cf3cc22c08d750a6", size = 3744674, upload-time = "2026-04-01T03:51:27.311Z" }, + { url = "https://files.pythonhosted.org/packages/fe/0b/7c63eb742859a6243f42288441f65ac9dac96ea59f409e43b713aafbe867/numba-0.65.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af143d823624033a128b5950c0aaf9ffc2386dfe954eb757119cf0432335534c", size = 3450620, upload-time = "2026-04-01T03:51:29.092Z" }, + { url = "https://files.pythonhosted.org/packages/73/36/88406bd58600cc696417b8e5dd6a056478da808f3eaf48d18e2421e0c2d9/numba-0.65.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a52d92ffd297c10364bce60cd1fcb88f99284ab5df085f2c6bcd1cb33b529a6f", size = 3801411, upload-time = "2026-04-01T03:51:34.321Z" }, + { url = "https://files.pythonhosted.org/packages/0c/61/ce753a1d7646dd477e16d15e89473703faebb8995d2f71d7ad69a540b565/numba-0.65.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da8e371e328c06d0010c3d8b44b21858652831b85bcfba78cb22c042e22dbd8e", size = 3501622, upload-time = "2026-04-01T03:51:36.348Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8f/3d116e4b8e92f6abace431afa4b2b944f4d65bdee83af886f5c4b263df95/numba-0.65.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b8a9008411615c69d083d1dcf477f75a5aa727b30beb16e139799e2be945cdfd", size = 3809537, upload-time = "2026-04-01T03:51:41.42Z" }, + { url = "https://files.pythonhosted.org/packages/b5/2c/6a3ca4128e253cb67affe06deb47688f51ce968f5111e2a06d010e6f1fa6/numba-0.65.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:af96c0cba53664efcb361528b8c75e011a6556c859c7e08424c2715201c6cf7a", size = 3508615, upload-time = "2026-04-01T03:51:43.444Z" }, + { url = "https://files.pythonhosted.org/packages/24/8d/e12d6ff4b9119db3cbf7b2db1ce257576441bd3c76388c786dea74f20b02/numba-0.65.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:05c0a9fdf75d85f57dee47b719e8d6415707b80aae45d75f63f9dc1b935c29f7", size = 3778456, upload-time = "2026-04-01T03:51:48.552Z" }, + { url = "https://files.pythonhosted.org/packages/17/89/abcd83e76f6a773276fe76244140671bcc5bf820f6e2ae1a15362ae4c8c9/numba-0.65.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:583680e0e8faf124d362df23b4b593f3221a8996341a63d1b664c122401bec2f", size = 3478464, upload-time = "2026-04-01T03:51:50.527Z" }, + { url = "https://files.pythonhosted.org/packages/ff/e5/8267b0adb0c01b52b553df5062fbbb42c30ed5362d08b85cc913a36f838f/numba-0.65.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c7fa502960f7a2f3f5cb025bc7bff888a3551277b92431bfdc5ba2f11a375749", size = 3816373, upload-time = "2026-04-01T03:51:56.18Z" }, + { url = "https://files.pythonhosted.org/packages/b0/f5/b8397ca360971669a93706b9274592b6864e4367a37d498fbbcb62aa2d48/numba-0.65.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5046c63f783ca3eb6195f826a50797465e7c4ce811daa17c9bea47e310c9b964", size = 3532782, upload-time = "2026-04-01T03:51:58.387Z" }, +] + +[[package]] +name = "numpy" +version = "2.3.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform != 'darwin'", + "python_full_version < '3.12' and sys_platform == 'darwin'", + "python_full_version < '3.12' and sys_platform != 'darwin'", +] +sdist = { url = "https://files.pythonhosted.org/packages/76/65/21b3bc86aac7b8f2862db1e808f1ea22b028e30a225a34a5ede9bf8678f2/numpy-2.3.5.tar.gz", hash = "sha256:784db1dcdab56bf0517743e746dfb0f885fc68d948aba86eeec2cba234bdf1c0", size = 20584950, upload-time = "2025-11-16T22:52:42.067Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/77/84dd1d2e34d7e2792a236ba180b5e8fcc1e3e414e761ce0253f63d7f572e/numpy-2.3.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:de5672f4a7b200c15a4127042170a694d4df43c992948f5e1af57f0174beed10", size = 17034641, upload-time = "2025-11-16T22:49:19.336Z" }, + { url = "https://files.pythonhosted.org/packages/2a/ea/25e26fa5837106cde46ae7d0b667e20f69cbbc0efd64cba8221411ab26ae/numpy-2.3.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:acfd89508504a19ed06ef963ad544ec6664518c863436306153e13e94605c218", size = 12528324, upload-time = "2025-11-16T22:49:22.582Z" }, + { url = "https://files.pythonhosted.org/packages/4d/1a/e85f0eea4cf03d6a0228f5c0256b53f2df4bc794706e7df019fc622e47f1/numpy-2.3.5-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ffe22d2b05504f786c867c8395de703937f934272eb67586817b46188b4ded6d", size = 5356872, upload-time = "2025-11-16T22:49:25.408Z" }, + { url = "https://files.pythonhosted.org/packages/5c/bb/35ef04afd567f4c989c2060cde39211e4ac5357155c1833bcd1166055c61/numpy-2.3.5-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:872a5cf366aec6bb1147336480fef14c9164b154aeb6542327de4970282cd2f5", size = 6893148, upload-time = "2025-11-16T22:49:27.549Z" }, + { url = "https://files.pythonhosted.org/packages/f2/2b/05bbeb06e2dff5eab512dfc678b1cc5ee94d8ac5956a0885c64b6b26252b/numpy-2.3.5-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3095bdb8dd297e5920b010e96134ed91d852d81d490e787beca7e35ae1d89cf7", size = 14557282, upload-time = "2025-11-16T22:49:30.964Z" }, + { url = "https://files.pythonhosted.org/packages/65/fb/2b23769462b34398d9326081fad5655198fcf18966fcb1f1e49db44fbf31/numpy-2.3.5-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8cba086a43d54ca804ce711b2a940b16e452807acebe7852ff327f1ecd49b0d4", size = 16897903, upload-time = "2025-11-16T22:49:34.191Z" }, + { url = "https://files.pythonhosted.org/packages/ac/14/085f4cf05fc3f1e8aa95e85404e984ffca9b2275a5dc2b1aae18a67538b8/numpy-2.3.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6cf9b429b21df6b99f4dee7a1218b8b7ffbbe7df8764dc0bd60ce8a0708fed1e", size = 16341672, upload-time = "2025-11-16T22:49:37.2Z" }, + { url = "https://files.pythonhosted.org/packages/6f/3b/1f73994904142b2aa290449b3bb99772477b5fd94d787093e4f24f5af763/numpy-2.3.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:396084a36abdb603546b119d96528c2f6263921c50df3c8fd7cb28873a237748", size = 18838896, upload-time = "2025-11-16T22:49:39.727Z" }, + { url = "https://files.pythonhosted.org/packages/cd/b9/cf6649b2124f288309ffc353070792caf42ad69047dcc60da85ee85fea58/numpy-2.3.5-cp311-cp311-win32.whl", hash = "sha256:b0c7088a73aef3d687c4deef8452a3ac7c1be4e29ed8bf3b366c8111128ac60c", size = 6563608, upload-time = "2025-11-16T22:49:42.079Z" }, + { url = "https://files.pythonhosted.org/packages/aa/44/9fe81ae1dcc29c531843852e2874080dc441338574ccc4306b39e2ff6e59/numpy-2.3.5-cp311-cp311-win_amd64.whl", hash = "sha256:a414504bef8945eae5f2d7cb7be2d4af77c5d1cb5e20b296c2c25b61dff2900c", size = 13078442, upload-time = "2025-11-16T22:49:43.99Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a7/f99a41553d2da82a20a2f22e93c94f928e4490bb447c9ff3c4ff230581d3/numpy-2.3.5-cp311-cp311-win_arm64.whl", hash = "sha256:0cd00b7b36e35398fa2d16af7b907b65304ef8bb4817a550e06e5012929830fa", size = 10458555, upload-time = "2025-11-16T22:49:47.092Z" }, + { url = "https://files.pythonhosted.org/packages/44/37/e669fe6cbb2b96c62f6bbedc6a81c0f3b7362f6a59230b23caa673a85721/numpy-2.3.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:74ae7b798248fe62021dbf3c914245ad45d1a6b0cb4a29ecb4b31d0bfbc4cc3e", size = 16733873, upload-time = "2025-11-16T22:49:49.84Z" }, + { url = "https://files.pythonhosted.org/packages/c5/65/df0db6c097892c9380851ab9e44b52d4f7ba576b833996e0080181c0c439/numpy-2.3.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ee3888d9ff7c14604052b2ca5535a30216aa0a58e948cdd3eeb8d3415f638769", size = 12259838, upload-time = "2025-11-16T22:49:52.863Z" }, + { url = "https://files.pythonhosted.org/packages/5b/e1/1ee06e70eb2136797abe847d386e7c0e830b67ad1d43f364dd04fa50d338/numpy-2.3.5-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:612a95a17655e213502f60cfb9bf9408efdc9eb1d5f50535cc6eb365d11b42b5", size = 5088378, upload-time = "2025-11-16T22:49:55.055Z" }, + { url = "https://files.pythonhosted.org/packages/6d/9c/1ca85fb86708724275103b81ec4cf1ac1d08f465368acfc8da7ab545bdae/numpy-2.3.5-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:3101e5177d114a593d79dd79658650fe28b5a0d8abeb8ce6f437c0e6df5be1a4", size = 6628559, upload-time = "2025-11-16T22:49:57.371Z" }, + { url = "https://files.pythonhosted.org/packages/74/78/fcd41e5a0ce4f3f7b003da85825acddae6d7ecb60cf25194741b036ca7d6/numpy-2.3.5-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b973c57ff8e184109db042c842423ff4f60446239bd585a5131cc47f06f789d", size = 14250702, upload-time = "2025-11-16T22:49:59.632Z" }, + { url = "https://files.pythonhosted.org/packages/b6/23/2a1b231b8ff672b4c450dac27164a8b2ca7d9b7144f9c02d2396518352eb/numpy-2.3.5-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0d8163f43acde9a73c2a33605353a4f1bc4798745a8b1d73183b28e5b435ae28", size = 16606086, upload-time = "2025-11-16T22:50:02.127Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c5/5ad26fbfbe2012e190cc7d5003e4d874b88bb18861d0829edc140a713021/numpy-2.3.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:51c1e14eb1e154ebd80e860722f9e6ed6ec89714ad2db2d3aa33c31d7c12179b", size = 16025985, upload-time = "2025-11-16T22:50:04.536Z" }, + { url = "https://files.pythonhosted.org/packages/d2/fa/dd48e225c46c819288148d9d060b047fd2a6fb1eb37eae25112ee4cb4453/numpy-2.3.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b46b4ec24f7293f23adcd2d146960559aaf8020213de8ad1909dba6c013bf89c", size = 18542976, upload-time = "2025-11-16T22:50:07.557Z" }, + { url = "https://files.pythonhosted.org/packages/05/79/ccbd23a75862d95af03d28b5c6901a1b7da4803181513d52f3b86ed9446e/numpy-2.3.5-cp312-cp312-win32.whl", hash = "sha256:3997b5b3c9a771e157f9aae01dd579ee35ad7109be18db0e85dbdbe1de06e952", size = 6285274, upload-time = "2025-11-16T22:50:10.746Z" }, + { url = "https://files.pythonhosted.org/packages/2d/57/8aeaf160312f7f489dea47ab61e430b5cb051f59a98ae68b7133ce8fa06a/numpy-2.3.5-cp312-cp312-win_amd64.whl", hash = "sha256:86945f2ee6d10cdfd67bcb4069c1662dd711f7e2a4343db5cecec06b87cf31aa", size = 12782922, upload-time = "2025-11-16T22:50:12.811Z" }, + { url = "https://files.pythonhosted.org/packages/78/a6/aae5cc2ca78c45e64b9ef22f089141d661516856cf7c8a54ba434576900d/numpy-2.3.5-cp312-cp312-win_arm64.whl", hash = "sha256:f28620fe26bee16243be2b7b874da327312240a7cdc38b769a697578d2100013", size = 10194667, upload-time = "2025-11-16T22:50:16.16Z" }, + { url = "https://files.pythonhosted.org/packages/db/69/9cde09f36da4b5a505341180a3f2e6fadc352fd4d2b7096ce9778db83f1a/numpy-2.3.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:d0f23b44f57077c1ede8c5f26b30f706498b4862d3ff0a7298b8411dd2f043ff", size = 16728251, upload-time = "2025-11-16T22:50:19.013Z" }, + { url = "https://files.pythonhosted.org/packages/79/fb/f505c95ceddd7027347b067689db71ca80bd5ecc926f913f1a23e65cf09b/numpy-2.3.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:aa5bc7c5d59d831d9773d1170acac7893ce3a5e130540605770ade83280e7188", size = 12254652, upload-time = "2025-11-16T22:50:21.487Z" }, + { url = "https://files.pythonhosted.org/packages/78/da/8c7738060ca9c31b30e9301ee0cf6c5ffdbf889d9593285a1cead337f9a5/numpy-2.3.5-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:ccc933afd4d20aad3c00bcef049cb40049f7f196e0397f1109dba6fed63267b0", size = 5083172, upload-time = "2025-11-16T22:50:24.562Z" }, + { url = "https://files.pythonhosted.org/packages/a4/b4/ee5bb2537fb9430fd2ef30a616c3672b991a4129bb1c7dcc42aa0abbe5d7/numpy-2.3.5-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:afaffc4393205524af9dfa400fa250143a6c3bc646c08c9f5e25a9f4b4d6a903", size = 6622990, upload-time = "2025-11-16T22:50:26.47Z" }, + { url = "https://files.pythonhosted.org/packages/95/03/dc0723a013c7d7c19de5ef29e932c3081df1c14ba582b8b86b5de9db7f0f/numpy-2.3.5-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9c75442b2209b8470d6d5d8b1c25714270686f14c749028d2199c54e29f20b4d", size = 14248902, upload-time = "2025-11-16T22:50:28.861Z" }, + { url = "https://files.pythonhosted.org/packages/f5/10/ca162f45a102738958dcec8023062dad0cbc17d1ab99d68c4e4a6c45fb2b/numpy-2.3.5-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11e06aa0af8c0f05104d56450d6093ee639e15f24ecf62d417329d06e522e017", size = 16597430, upload-time = "2025-11-16T22:50:31.56Z" }, + { url = "https://files.pythonhosted.org/packages/2a/51/c1e29be863588db58175175f057286900b4b3327a1351e706d5e0f8dd679/numpy-2.3.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ed89927b86296067b4f81f108a2271d8926467a8868e554eaf370fc27fa3ccaf", size = 16024551, upload-time = "2025-11-16T22:50:34.242Z" }, + { url = "https://files.pythonhosted.org/packages/83/68/8236589d4dbb87253d28259d04d9b814ec0ecce7cb1c7fed29729f4c3a78/numpy-2.3.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51c55fe3451421f3a6ef9a9c1439e82101c57a2c9eab9feb196a62b1a10b58ce", size = 18533275, upload-time = "2025-11-16T22:50:37.651Z" }, + { url = "https://files.pythonhosted.org/packages/40/56/2932d75b6f13465239e3b7b7e511be27f1b8161ca2510854f0b6e521c395/numpy-2.3.5-cp313-cp313-win32.whl", hash = "sha256:1978155dd49972084bd6ef388d66ab70f0c323ddee6f693d539376498720fb7e", size = 6277637, upload-time = "2025-11-16T22:50:40.11Z" }, + { url = "https://files.pythonhosted.org/packages/0c/88/e2eaa6cffb115b85ed7c7c87775cb8bcf0816816bc98ca8dbfa2ee33fe6e/numpy-2.3.5-cp313-cp313-win_amd64.whl", hash = "sha256:00dc4e846108a382c5869e77c6ed514394bdeb3403461d25a829711041217d5b", size = 12779090, upload-time = "2025-11-16T22:50:42.503Z" }, + { url = "https://files.pythonhosted.org/packages/8f/88/3f41e13a44ebd4034ee17baa384acac29ba6a4fcc2aca95f6f08ca0447d1/numpy-2.3.5-cp313-cp313-win_arm64.whl", hash = "sha256:0472f11f6ec23a74a906a00b48a4dcf3849209696dff7c189714511268d103ae", size = 10194710, upload-time = "2025-11-16T22:50:44.971Z" }, + { url = "https://files.pythonhosted.org/packages/13/cb/71744144e13389d577f867f745b7df2d8489463654a918eea2eeb166dfc9/numpy-2.3.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:414802f3b97f3c1eef41e530aaba3b3c1620649871d8cb38c6eaff034c2e16bd", size = 16827292, upload-time = "2025-11-16T22:50:47.715Z" }, + { url = "https://files.pythonhosted.org/packages/71/80/ba9dc6f2a4398e7f42b708a7fdc841bb638d353be255655498edbf9a15a8/numpy-2.3.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5ee6609ac3604fa7780e30a03e5e241a7956f8e2fcfe547d51e3afa5247ac47f", size = 12378897, upload-time = "2025-11-16T22:50:51.327Z" }, + { url = "https://files.pythonhosted.org/packages/2e/6d/db2151b9f64264bcceccd51741aa39b50150de9b602d98ecfe7e0c4bff39/numpy-2.3.5-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:86d835afea1eaa143012a2d7a3f45a3adce2d7adc8b4961f0b362214d800846a", size = 5207391, upload-time = "2025-11-16T22:50:54.542Z" }, + { url = "https://files.pythonhosted.org/packages/80/ae/429bacace5ccad48a14c4ae5332f6aa8ab9f69524193511d60ccdfdc65fa/numpy-2.3.5-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:30bc11310e8153ca664b14c5f1b73e94bd0503681fcf136a163de856f3a50139", size = 6721275, upload-time = "2025-11-16T22:50:56.794Z" }, + { url = "https://files.pythonhosted.org/packages/74/5b/1919abf32d8722646a38cd527bc3771eb229a32724ee6ba340ead9b92249/numpy-2.3.5-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1062fde1dcf469571705945b0f221b73928f34a20c904ffb45db101907c3454e", size = 14306855, upload-time = "2025-11-16T22:50:59.208Z" }, + { url = "https://files.pythonhosted.org/packages/a5/87/6831980559434973bebc30cd9c1f21e541a0f2b0c280d43d3afd909b66d0/numpy-2.3.5-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce581db493ea1a96c0556360ede6607496e8bf9b3a8efa66e06477267bc831e9", size = 16657359, upload-time = "2025-11-16T22:51:01.991Z" }, + { url = "https://files.pythonhosted.org/packages/dd/91/c797f544491ee99fd00495f12ebb7802c440c1915811d72ac5b4479a3356/numpy-2.3.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:cc8920d2ec5fa99875b670bb86ddeb21e295cb07aa331810d9e486e0b969d946", size = 16093374, upload-time = "2025-11-16T22:51:05.291Z" }, + { url = "https://files.pythonhosted.org/packages/74/a6/54da03253afcbe7a72785ec4da9c69fb7a17710141ff9ac5fcb2e32dbe64/numpy-2.3.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9ee2197ef8c4f0dfe405d835f3b6a14f5fee7782b5de51ba06fb65fc9b36e9f1", size = 18594587, upload-time = "2025-11-16T22:51:08.585Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/aff53abbdd41b0ecca94285f325aff42357c6b5abc482a3fcb4994290b18/numpy-2.3.5-cp313-cp313t-win32.whl", hash = "sha256:70b37199913c1bd300ff6e2693316c6f869c7ee16378faf10e4f5e3275b299c3", size = 6405940, upload-time = "2025-11-16T22:51:11.541Z" }, + { url = "https://files.pythonhosted.org/packages/d5/81/50613fec9d4de5480de18d4f8ef59ad7e344d497edbef3cfd80f24f98461/numpy-2.3.5-cp313-cp313t-win_amd64.whl", hash = "sha256:b501b5fa195cc9e24fe102f21ec0a44dffc231d2af79950b451e0d99cea02234", size = 12920341, upload-time = "2025-11-16T22:51:14.312Z" }, + { url = "https://files.pythonhosted.org/packages/bb/ab/08fd63b9a74303947f34f0bd7c5903b9c5532c2d287bead5bdf4c556c486/numpy-2.3.5-cp313-cp313t-win_arm64.whl", hash = "sha256:a80afd79f45f3c4a7d341f13acbe058d1ca8ac017c165d3fa0d3de6bc1a079d7", size = 10262507, upload-time = "2025-11-16T22:51:16.846Z" }, + { url = "https://files.pythonhosted.org/packages/ba/97/1a914559c19e32d6b2e233cf9a6a114e67c856d35b1d6babca571a3e880f/numpy-2.3.5-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:bf06bc2af43fa8d32d30fae16ad965663e966b1a3202ed407b84c989c3221e82", size = 16735706, upload-time = "2025-11-16T22:51:19.558Z" }, + { url = "https://files.pythonhosted.org/packages/57/d4/51233b1c1b13ecd796311216ae417796b88b0616cfd8a33ae4536330748a/numpy-2.3.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:052e8c42e0c49d2575621c158934920524f6c5da05a1d3b9bab5d8e259e045f0", size = 12264507, upload-time = "2025-11-16T22:51:22.492Z" }, + { url = "https://files.pythonhosted.org/packages/45/98/2fe46c5c2675b8306d0b4a3ec3494273e93e1226a490f766e84298576956/numpy-2.3.5-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:1ed1ec893cff7040a02c8aa1c8611b94d395590d553f6b53629a4461dc7f7b63", size = 5093049, upload-time = "2025-11-16T22:51:25.171Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0e/0698378989bb0ac5f1660c81c78ab1fe5476c1a521ca9ee9d0710ce54099/numpy-2.3.5-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:2dcd0808a421a482a080f89859a18beb0b3d1e905b81e617a188bd80422d62e9", size = 6626603, upload-time = "2025-11-16T22:51:27Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a6/9ca0eecc489640615642a6cbc0ca9e10df70df38c4d43f5a928ff18d8827/numpy-2.3.5-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:727fd05b57df37dc0bcf1a27767a3d9a78cbbc92822445f32cc3436ba797337b", size = 14262696, upload-time = "2025-11-16T22:51:29.402Z" }, + { url = "https://files.pythonhosted.org/packages/c8/f6/07ec185b90ec9d7217a00eeeed7383b73d7e709dae2a9a021b051542a708/numpy-2.3.5-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fffe29a1ef00883599d1dc2c51aa2e5d80afe49523c261a74933df395c15c520", size = 16597350, upload-time = "2025-11-16T22:51:32.167Z" }, + { url = "https://files.pythonhosted.org/packages/75/37/164071d1dde6a1a84c9b8e5b414fa127981bad47adf3a6b7e23917e52190/numpy-2.3.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8f7f0e05112916223d3f438f293abf0727e1181b5983f413dfa2fefc4098245c", size = 16040190, upload-time = "2025-11-16T22:51:35.403Z" }, + { url = "https://files.pythonhosted.org/packages/08/3c/f18b82a406b04859eb026d204e4e1773eb41c5be58410f41ffa511d114ae/numpy-2.3.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2e2eb32ddb9ccb817d620ac1d8dae7c3f641c1e5f55f531a33e8ab97960a75b8", size = 18536749, upload-time = "2025-11-16T22:51:39.698Z" }, + { url = "https://files.pythonhosted.org/packages/40/79/f82f572bf44cf0023a2fe8588768e23e1592585020d638999f15158609e1/numpy-2.3.5-cp314-cp314-win32.whl", hash = "sha256:66f85ce62c70b843bab1fb14a05d5737741e74e28c7b8b5a064de10142fad248", size = 6335432, upload-time = "2025-11-16T22:51:42.476Z" }, + { url = "https://files.pythonhosted.org/packages/a3/2e/235b4d96619931192c91660805e5e49242389742a7a82c27665021db690c/numpy-2.3.5-cp314-cp314-win_amd64.whl", hash = "sha256:e6a0bc88393d65807d751a614207b7129a310ca4fe76a74e5c7da5fa5671417e", size = 12919388, upload-time = "2025-11-16T22:51:45.275Z" }, + { url = "https://files.pythonhosted.org/packages/07/2b/29fd75ce45d22a39c61aad74f3d718e7ab67ccf839ca8b60866054eb15f8/numpy-2.3.5-cp314-cp314-win_arm64.whl", hash = "sha256:aeffcab3d4b43712bb7a60b65f6044d444e75e563ff6180af8f98dd4b905dfd2", size = 10476651, upload-time = "2025-11-16T22:51:47.749Z" }, + { url = "https://files.pythonhosted.org/packages/17/e1/f6a721234ebd4d87084cfa68d081bcba2f5cfe1974f7de4e0e8b9b2a2ba1/numpy-2.3.5-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:17531366a2e3a9e30762c000f2c43a9aaa05728712e25c11ce1dbe700c53ad41", size = 16834503, upload-time = "2025-11-16T22:51:50.443Z" }, + { url = "https://files.pythonhosted.org/packages/5c/1c/baf7ffdc3af9c356e1c135e57ab7cf8d247931b9554f55c467efe2c69eff/numpy-2.3.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d21644de1b609825ede2f48be98dfde4656aefc713654eeee280e37cadc4e0ad", size = 12381612, upload-time = "2025-11-16T22:51:53.609Z" }, + { url = "https://files.pythonhosted.org/packages/74/91/f7f0295151407ddc9ba34e699013c32c3c91944f9b35fcf9281163dc1468/numpy-2.3.5-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:c804e3a5aba5460c73955c955bdbd5c08c354954e9270a2c1565f62e866bdc39", size = 5210042, upload-time = "2025-11-16T22:51:56.213Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3b/78aebf345104ec50dd50a4d06ddeb46a9ff5261c33bcc58b1c4f12f85ec2/numpy-2.3.5-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:cc0a57f895b96ec78969c34f682c602bf8da1a0270b09bc65673df2e7638ec20", size = 6724502, upload-time = "2025-11-16T22:51:58.584Z" }, + { url = "https://files.pythonhosted.org/packages/02/c6/7c34b528740512e57ef1b7c8337ab0b4f0bddf34c723b8996c675bc2bc91/numpy-2.3.5-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:900218e456384ea676e24ea6a0417f030a3b07306d29d7ad843957b40a9d8d52", size = 14308962, upload-time = "2025-11-16T22:52:01.698Z" }, + { url = "https://files.pythonhosted.org/packages/80/35/09d433c5262bc32d725bafc619e095b6a6651caf94027a03da624146f655/numpy-2.3.5-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:09a1bea522b25109bf8e6f3027bd810f7c1085c64a0c7ce050c1676ad0ba010b", size = 16655054, upload-time = "2025-11-16T22:52:04.267Z" }, + { url = "https://files.pythonhosted.org/packages/7a/ab/6a7b259703c09a88804fa2430b43d6457b692378f6b74b356155283566ac/numpy-2.3.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:04822c00b5fd0323c8166d66c701dc31b7fbd252c100acd708c48f763968d6a3", size = 16091613, upload-time = "2025-11-16T22:52:08.651Z" }, + { url = "https://files.pythonhosted.org/packages/c2/88/330da2071e8771e60d1038166ff9d73f29da37b01ec3eb43cb1427464e10/numpy-2.3.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d6889ec4ec662a1a37eb4b4fb26b6100841804dac55bd9df579e326cdc146227", size = 18591147, upload-time = "2025-11-16T22:52:11.453Z" }, + { url = "https://files.pythonhosted.org/packages/51/41/851c4b4082402d9ea860c3626db5d5df47164a712cb23b54be028b184c1c/numpy-2.3.5-cp314-cp314t-win32.whl", hash = "sha256:93eebbcf1aafdf7e2ddd44c2923e2672e1010bddc014138b229e49725b4d6be5", size = 6479806, upload-time = "2025-11-16T22:52:14.641Z" }, + { url = "https://files.pythonhosted.org/packages/90/30/d48bde1dfd93332fa557cff1972fbc039e055a52021fbef4c2c4b1eefd17/numpy-2.3.5-cp314-cp314t-win_amd64.whl", hash = "sha256:c8a9958e88b65c3b27e22ca2a076311636850b612d6bbfb76e8d156aacde2aaf", size = 13105760, upload-time = "2025-11-16T22:52:17.975Z" }, + { url = "https://files.pythonhosted.org/packages/2d/fd/4b5eb0b3e888d86aee4d198c23acec7d214baaf17ea93c1adec94c9518b9/numpy-2.3.5-cp314-cp314t-win_arm64.whl", hash = "sha256:6203fdf9f3dc5bdaed7319ad8698e685c7a3be10819f41d32a0723e611733b42", size = 10545459, upload-time = "2025-11-16T22:52:20.55Z" }, + { url = "https://files.pythonhosted.org/packages/c6/65/f9dea8e109371ade9c782b4e4756a82edf9d3366bca495d84d79859a0b79/numpy-2.3.5-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:f0963b55cdd70fad460fa4c1341f12f976bb26cb66021a5580329bd498988310", size = 16910689, upload-time = "2025-11-16T22:52:23.247Z" }, + { url = "https://files.pythonhosted.org/packages/00/4f/edb00032a8fb92ec0a679d3830368355da91a69cab6f3e9c21b64d0bb986/numpy-2.3.5-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f4255143f5160d0de972d28c8f9665d882b5f61309d8362fdd3e103cf7bf010c", size = 12457053, upload-time = "2025-11-16T22:52:26.367Z" }, + { url = "https://files.pythonhosted.org/packages/16/a4/e8a53b5abd500a63836a29ebe145fc1ab1f2eefe1cfe59276020373ae0aa/numpy-2.3.5-pp311-pypy311_pp73-macosx_14_0_arm64.whl", hash = "sha256:a4b9159734b326535f4dd01d947f919c6eefd2d9827466a696c44ced82dfbc18", size = 5285635, upload-time = "2025-11-16T22:52:29.266Z" }, + { url = "https://files.pythonhosted.org/packages/a3/2f/37eeb9014d9c8b3e9c55bc599c68263ca44fdbc12a93e45a21d1d56df737/numpy-2.3.5-pp311-pypy311_pp73-macosx_14_0_x86_64.whl", hash = "sha256:2feae0d2c91d46e59fcd62784a3a83b3fb677fead592ce51b5a6fbb4f95965ff", size = 6801770, upload-time = "2025-11-16T22:52:31.421Z" }, + { url = "https://files.pythonhosted.org/packages/7d/e4/68d2f474df2cb671b2b6c2986a02e520671295647dad82484cde80ca427b/numpy-2.3.5-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ffac52f28a7849ad7576293c0cb7b9f08304e8f7d738a8cb8a90ec4c55a998eb", size = 14391768, upload-time = "2025-11-16T22:52:33.593Z" }, + { url = "https://files.pythonhosted.org/packages/b8/50/94ccd8a2b141cb50651fddd4f6a48874acb3c91c8f0842b08a6afc4b0b21/numpy-2.3.5-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63c0e9e7eea69588479ebf4a8a270d5ac22763cc5854e9a7eae952a3908103f7", size = 16729263, upload-time = "2025-11-16T22:52:36.369Z" }, + { url = "https://files.pythonhosted.org/packages/2d/ee/346fa473e666fe14c52fcdd19ec2424157290a032d4c41f98127bfb31ac7/numpy-2.3.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f16417ec91f12f814b10bafe79ef77e70113a2f5f7018640e7425ff979253425", size = 12967213, upload-time = "2025-11-16T22:52:39.38Z" }, +] + [[package]] name = "numpy" version = "2.4.2" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14' and sys_platform == 'darwin'", + "python_full_version >= '3.14' and sys_platform != 'darwin'", + "python_full_version == '3.13.*' and sys_platform == 'darwin'", + "python_full_version == '3.13.*' and sys_platform != 'darwin'", +] sdist = { url = "https://files.pythonhosted.org/packages/57/fd/0005efbd0af48e55eb3c7208af93f2862d4b1a56cd78e84309a2d959208d/numpy-2.4.2.tar.gz", hash = "sha256:659a6107e31a83c4e33f763942275fd278b21d095094044eb35569e86a21ddae", size = 20723651, upload-time = "2026-01-31T23:13:10.135Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/d3/44/71852273146957899753e69986246d6a176061ea183407e95418c2aa4d9a/numpy-2.4.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e7e88598032542bd49af7c4747541422884219056c268823ef6e5e89851c8825", size = 16955478, upload-time = "2026-01-31T23:10:25.623Z" }, @@ -2671,6 +3773,323 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/e5/b7d20451657664b07986c2f6e3be564433f5dcaf3482d68eaecd79afaf03/numpy-2.4.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:be71bf1edb48ebbbf7f6337b5bfd2f895d1902f6335a5830b20141fc126ffba0", size = 12502577, upload-time = "2026-01-31T23:13:07.08Z" }, ] +[[package]] +name = "nvidia-cublas" +version = "13.1.0.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/a5/fce49e2ae977e0ccc084e5adafceb4f0ac0c8333cb6863501618a7277f67/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c86fc7f7ae36d7528288c5d88098edcb7b02c633d262e7ddbb86b0ad91be5df2", size = 542851226, upload-time = "2025-10-09T08:59:04.818Z" }, + { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236, upload-time = "2025-10-09T08:59:32.536Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, +] + +[[package]] +name = "nvidia-cuda-nvdisasm" +version = "13.3.73" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/be/e9de501cb71b10f7654381a485fa4ebf470ea25c3dce018cccaecf8a8f9a/nvidia_cuda_nvdisasm-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:dd4751884f9016b9b6dbf007abdeb5681d0a2edc731dd3d2fda9d6d878e88f73", size = 4744517, upload-time = "2026-06-29T16:48:33.527Z" }, + { url = "https://files.pythonhosted.org/packages/86/3e/88460ebd737e559e8e9843db7a63f8ced9ec7be1882344438819dd13aebc/nvidia_cuda_nvdisasm-13.3.73-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fa17084b07c0dca68a42892f771b4b1b40fbe9b91660209623e61cea611cae8c", size = 4782824, upload-time = "2026-06-29T16:49:05.109Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, + { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.19.0.56" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, + { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321, upload-time = "2026-02-03T20:44:52.837Z" }, +] + +[[package]] +name = "nvidia-cudnn-frontend" +version = "1.18.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/9a/83d3d080118de4a7810fa019349edec634b8b37b9cafaacd05719de62dd6/nvidia_cudnn_frontend-1.18.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6d4d0b88d617b233a503c84980b54d840b60b2734497d1a7a071ec5293daec2", size = 2023709, upload-time = "2026-01-27T23:32:10.912Z" }, + { url = "https://files.pythonhosted.org/packages/13/c7/c3624b3ed77b102618f26295e816b27f1c3ebb1143730237a9f51d403c3f/nvidia_cudnn_frontend-1.18.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:382ea063b92cbfd5b442cb75ff8422932d78276aecf139e46713ed1ad3d07af4", size = 2155568, upload-time = "2026-01-27T23:07:13.277Z" }, + { url = "https://files.pythonhosted.org/packages/e3/b4/604e230378680ee117849a4e1045baca092f93161a829291a84d5acce70c/nvidia_cudnn_frontend-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:310b417f2848a83d1437203fcaeea320a74fb7f28af20bf42bf5afc9c01f1c12", size = 2027408, upload-time = "2026-01-27T23:32:46.576Z" }, + { url = "https://files.pythonhosted.org/packages/c6/52/08f98262e77b1cbcc834cc1a5db494d0661ea1dbdea58c2e2d51a57fdaca/nvidia_cudnn_frontend-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c023539ca6de99234cf5102c3ec0d6af817f5396fc93028a22ba5b834a35b8a", size = 2159245, upload-time = "2026-01-27T23:07:32.664Z" }, + { url = "https://files.pythonhosted.org/packages/e8/bd/db791a26ebb6a6e1268f518e18c82d8ad18546f7008f4b0d5bde15f927de/nvidia_cudnn_frontend-1.18.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a6e2b7bd43705ffa4af3b187374fdd5e7d09fc228a4d65fc8b4b0a537a8e605", size = 2027249, upload-time = "2026-01-27T23:33:22.46Z" }, + { url = "https://files.pythonhosted.org/packages/19/74/3038cf496d5de7cfdff730f5202e438c17d9123de507059340e02ddff9d7/nvidia_cudnn_frontend-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0544206b02cae9da4f044ca3fe7416b99e0c8a8052285dd3e5a8fc445d34f9c", size = 2160001, upload-time = "2026-01-27T23:07:50.248Z" }, + { url = "https://files.pythonhosted.org/packages/a3/0a/515209dd2afc6027bf1112bf415f575bfe9628d18877abe7424cb597dd7b/nvidia_cudnn_frontend-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b489da1b30f1d7da822b37b89cc4f68afd80e020eb57e4ab24921f8b57f6e946", size = 2028689, upload-time = "2026-02-11T21:32:04.235Z" }, + { url = "https://files.pythonhosted.org/packages/ab/57/52d18e1f50979eeabfafb408ec73068afc5a1e1ccd21636240317cd456d4/nvidia_cudnn_frontend-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37688c81a34ac590aff9de4c34d2968bab949411af707baa327616ebd4b34ae1", size = 2160182, upload-time = "2026-02-11T21:25:18.437Z" }, +] + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, +] + +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, +] + +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, +] + +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu13" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/10/8dcd1175260706a2fc92a16a52e306b71d4c1ea0b0cc4a9484183399818a/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:400c6ed1cf6780fc6efedd64ec9f1345871767e6a1a0a552a1ea0578117ea77c", size = 220791277, upload-time = "2025-08-13T19:22:40.982Z" }, + { url = "https://files.pythonhosted.org/packages/fd/53/43b0d71f4e702fa9733f8b4571fdca50a8813f1e450b656c239beff12315/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25e30a8a7323935d4ad0340b95a0b69926eee755767e8e0b1cf8dd85b197d3fd", size = 169884119, upload-time = "2025-08-13T19:23:41.967Z" }, +] + +[[package]] +name = "nvidia-cutlass-dsl" +version = "4.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cutlass-dsl-libs-base" }, + { name = "nvidia-cutlass-dsl-libs-cu12" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/ec/14e6ecbfed31ec35bbd1bb6965ae61f879370906a8f9c2e851e292704ba4/nvidia_cutlass_dsl-4.6.2-py3-none-any.whl", hash = "sha256:06ac62deb182a852dfb053032cf945bf02b9aa1bf502a18b129d280d7babb26c", size = 10460, upload-time = "2026-08-05T14:39:41.657Z" }, +] + +[[package]] +name = "nvidia-cutlass-dsl-libs-base" +version = "4.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-python" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "nvidia-cuda-nvdisasm" }, + { name = "nvidia-cutlass-dsl-libs-core" }, + { name = "protobuf" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/0e/5020da76c7dd3ed74acc6f435e53ca2df967762b2d8408f3e81f146e018a/nvidia_cutlass_dsl_libs_base-4.6.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:aae76fcc94c5483279f06848ee470b87417b9b1a4ad8f822a12e2654cbaf513a", size = 3321401, upload-time = "2026-08-05T14:11:32.779Z" }, + { url = "https://files.pythonhosted.org/packages/12/ce/dfefb22eb438de1e0e6c6627f188449ad3f65448522ba80a12a67bb14715/nvidia_cutlass_dsl_libs_base-4.6.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:551097a99537ffdd239678088865ab8d1e273633f0961493509284d8445dd420", size = 2827063, upload-time = "2026-08-05T14:11:54.706Z" }, + { url = "https://files.pythonhosted.org/packages/aa/f3/4be98f2faa283471e355a42ecbc20956fb2c3b6dbc71861f483be277e99a/nvidia_cutlass_dsl_libs_base-4.6.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e129db7155d56ba4478be098c0f819e37690a6ce5d43a9a9a83bf1300d32c57f", size = 3321894, upload-time = "2026-08-05T14:12:30.139Z" }, + { url = "https://files.pythonhosted.org/packages/54/1b/cf4a0837054bb4b9dc36cdd86f9d2e7fead3629e3373aaa7f7ec1e5e1542/nvidia_cutlass_dsl_libs_base-4.6.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:24dfaddad6077fd0de14eecaf66a72762f2f5f30ae1872231f5bf8b243ac2eac", size = 2824987, upload-time = "2026-08-05T14:12:49.618Z" }, + { url = "https://files.pythonhosted.org/packages/41/e9/a402e079e926f1fbcf82e30dfcd87aa924aaf91b02db08d57e83b158dbb2/nvidia_cutlass_dsl_libs_base-4.6.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:334a9be3c5054286666b14f81c9278075c2007b947ba75f7ee44b94aadf26415", size = 3321794, upload-time = "2026-08-05T14:13:12.501Z" }, + { url = "https://files.pythonhosted.org/packages/d8/4c/0a6c6e785e8fb2bfc6a0941559d30fbaf6264ae77ee1a26a42e37fd6efdc/nvidia_cutlass_dsl_libs_base-4.6.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:896f8a87d0815e59066b0607930c9905a5cf8f5c7a01a423b0093bc876bac4b4", size = 2825075, upload-time = "2026-08-05T14:13:34.746Z" }, + { url = "https://files.pythonhosted.org/packages/47/9d/627165c9044426610176c4966ca8fd0fb9856684db918e725f6561872f8a/nvidia_cutlass_dsl_libs_base-4.6.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f35403515f5ba5bc69924f9faf0206af4bde3ebef9117d5f696529102cf4dbf7", size = 3321977, upload-time = "2026-08-05T14:14:01.683Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b8/50b0fedea6db2a522f299652d6187b74de1beb74f201a1533c29cec9fb67/nvidia_cutlass_dsl_libs_base-4.6.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:17a0165624144a2e6962d6e2322cd9008050326808b89a649877450e732403e4", size = 2825099, upload-time = "2026-08-05T14:14:22.171Z" }, + { url = "https://files.pythonhosted.org/packages/d9/91/a904783c7032da5c56a1e24f5daba5407018a356668798a71dbd69fc726a/nvidia_cutlass_dsl_libs_base-4.6.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:77266dc557bc2575244a19b1c43fc971252cf152b198cd872553c2734e6a4e44", size = 3330162, upload-time = "2026-08-05T14:14:57.698Z" }, + { url = "https://files.pythonhosted.org/packages/4b/bf/ce685d267cab2728ccacf7d47c8a52ab600fab6d67865d5be7e1e971bb8e/nvidia_cutlass_dsl_libs_base-4.6.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:b8fd31f484d6231ea11c29cee2aebeaa86e738e78684d09dc1d34be08069038a", size = 2837750, upload-time = "2026-08-05T14:15:23.647Z" }, +] + +[[package]] +name = "nvidia-cutlass-dsl-libs-core" +version = "4.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-python" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "nvidia-cuda-nvdisasm" }, + { name = "protobuf" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/ef/10c0089234a32f1379e1f213f28f8c7521a9ff0154e3acc58f613d5ef3be/nvidia_cutlass_dsl_libs_core-4.6.2-py3-none-any.whl", hash = "sha256:571d4b46fca1bfbb123d0dc348eaa7fd80af0014ddffc07b849e8195b2364333", size = 772393, upload-time = "2026-08-05T14:09:50.76Z" }, +] + +[[package]] +name = "nvidia-cutlass-dsl-libs-cu12" +version = "4.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-python" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "nvidia-cuda-nvdisasm" }, + { name = "nvidia-cutlass-dsl-libs-base" }, + { name = "protobuf" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/26/033408d2f1488e941b3434c21e703ec11c42873166af1eb8348271565782/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d245dfb3255d95d1ac75063b758acd3e40d5c6699dc98c80f488165260923e11", size = 87011888, upload-time = "2026-08-05T14:18:09.399Z" }, + { url = "https://files.pythonhosted.org/packages/1a/1b/4bc2842e0b497d9d0fda73976af7640ef742cdb5210e2d143b72c5ec3938/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:4ad5083bf8bcbda83973bdc38a03c2c8452d11fe0afc04b10fc362e23ddd7f8c", size = 88454141, upload-time = "2026-08-05T14:18:31.516Z" }, + { url = "https://files.pythonhosted.org/packages/ad/b7/019a8f74ad30ad9b6190dd5963d8921fc2b03777316e47260eb822624a0a/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ea3b96dd6b627fe20637acfa7876ea1adfa8d81a2b106ffff0bb2d042c0c2731", size = 87013242, upload-time = "2026-08-05T14:18:57.703Z" }, + { url = "https://files.pythonhosted.org/packages/80/6e/70c545a600b52b3f0fbbe01608aefc4e675c924ba886485ad72541f5c88e/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7f510307369d522da8e7d666557b9b9e7df06b94748bd5dc0198d59dfd0d918a", size = 88454261, upload-time = "2026-08-05T14:21:14.733Z" }, + { url = "https://files.pythonhosted.org/packages/e1/54/6006a2b4fcd05f32be451fe8881968dcbed526438b4eae9a527534062c79/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2f3b163060325777a3847954b3a599556b2bff500eca19806a81d1d635bb4149", size = 87012255, upload-time = "2026-08-05T14:21:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/d1/7a/a08ac0ecdd88bb129a56d95b7c58ea6826e6f5fa00191ef7ecdff8085836/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:84243743382948c83ca627c2dcb106db3844c31d45b8cc2af3d14bc9df0a535e", size = 88453568, upload-time = "2026-08-05T14:22:49.158Z" }, + { url = "https://files.pythonhosted.org/packages/c8/7e/e0f29110f85c129caf2ccf01f0d49e473e6d66c2a220f904870f6093d570/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:37f5d1a7eb8a00a6e303c9ed265ef2586075af49deb9e51865c38275fa1db802", size = 87012154, upload-time = "2026-08-05T14:23:09.963Z" }, + { url = "https://files.pythonhosted.org/packages/16/87/d8af4c8972baaa3b80cbb633fede58f4c8b0d35c8724f1cfa01027103bdb/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0dcb6dd53de145e2b1dde5b2d91a5c30145c97dac9eb18281209703ca9e00a5e", size = 88454125, upload-time = "2026-08-05T14:23:44.62Z" }, + { url = "https://files.pythonhosted.org/packages/26/d6/99ed69f57bde2ee6c1b0c1fd5440944b4ce80476d8799dd7212ea2d175dd/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:38f33d937dd375ee2fef7c802326a18e40bb9083669bb9d1d76462c778e72011", size = 87019480, upload-time = "2026-08-05T14:24:05.561Z" }, + { url = "https://files.pythonhosted.org/packages/46/f8/6b40afd27c89a7cf737c59393a4e9dd49da918ca8eab437f65a19cb3f912/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:06d8b367aec0ea1a8bca710b92d26df48a2378b16439b46ea8492938454b326a", size = 88464112, upload-time = "2026-08-05T14:24:28.062Z" }, +] + +[[package]] +name = "nvidia-ml-py" +version = "13.610.43" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f0/b5/a8fbc356f768fa5c9cfd646668fd7d34bf55bdd1c6e20754642a64d930d4/nvidia_ml_py-13.610.43.tar.gz", hash = "sha256:65437eb73d68d0c62c931ca4d45038472faff03bd0b8729abba4b899f70d60f2", size = 52109, upload-time = "2026-06-01T18:54:08.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/45/caa600acfab94560807a20a64b5830d2cd3c3202b7f1328644d70b7d6bd8/nvidia_ml_py-13.610.43-py3-none-any.whl", hash = "sha256:f13c72698edef492f985cc225f14faafe68ae065a2e407f45bdf6f4b9b43fde8", size = 53163, upload-time = "2026-06-01T18:54:07.704Z" }, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.28.9" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/55/1920646a2e43ffd4fc958536b276197ed740e9e0c54105b4bb3521591fc7/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:01c873ba1626b54caa12272ed228dc5b2781545e0ae8ba3f432a8ef1c6d78643", size = 196561677, upload-time = "2025-11-18T05:49:03.45Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b4/878fefaad5b2bcc6fcf8d474a25e3e3774bc5133e4b58adff4d0bca238bc/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:e4553a30f34195f3fa1da02a6da3d6337d28f2003943aa0a3d247bbc25fefc42", size = 196493177, upload-time = "2025-11-18T05:49:17.677Z" }, +] + +[[package]] +name = "nvidia-nvjitlink" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu13" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, +] + +[[package]] +name = "openai" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx2" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/8c/2f500e8be09d1ae98c530467962535198b02cd4550cd418bbbaedc8b2910/openai-3.0.0.tar.gz", hash = "sha256:ffd00ef1678d70957e1f1ed98d5bfcf1d661f41ea4482f22e7d0144a66435a49", size = 1123740, upload-time = "2026-08-12T01:55:50.849Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/0d/9850e7eddb5e66da4439ed503e78e09ad1fd0195e6df51e4236c75763581/openai-3.0.0-py3-none-any.whl", hash = "sha256:8d32ac3a6647a66910d6cb8a64f0fa5a6c823604b6e82db83d9d055c6709bd51", size = 1665775, upload-time = "2026-08-12T01:55:48.678Z" }, +] + +[[package]] +name = "openai-harmony" +version = "0.0.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3e/92/2d038d096f29179c7c9571b431f9e739f87a487121901725e23fe338dd9d/openai_harmony-0.0.8.tar.gz", hash = "sha256:6e43f98e6c242fa2de6f8ea12eab24af63fa2ed3e89c06341fb9d92632c5cbdf", size = 284777, upload-time = "2025-11-05T19:07:06.727Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/d2/ce6953ca87db9cae3e775024184da7d1c5cb88cead19a2d75b42f00a959c/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e4f709815924ec325b9a890e6ab2bbb0ceec8e319a4e257328eb752cf36b2efc", size = 2948463, upload-time = "2025-11-05T19:06:48.17Z" }, + { url = "https://files.pythonhosted.org/packages/fa/4c/b553c9651662d6ce102ca7f3629d268b23df1abe5841e24bed81e8a8e949/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5cfcfd963b50a41fc656c84d3440ca6eecdccd6c552158ce790b8f2e33dfb5a9", size = 2704083, upload-time = "2025-11-05T19:06:50.205Z" }, + { url = "https://files.pythonhosted.org/packages/9b/af/4eec8f9ab9c27bcdb444460c72cf43011d176fc44c79d6e113094ca1e152/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a3a16972aa1cee38ea958470cd04ac9a2d5ac38fdcf77ab686611246220c158", size = 2959765, upload-time = "2025-11-05T19:06:53.62Z" }, + { url = "https://files.pythonhosted.org/packages/11/3c/33f3374e4624e0e776f6b13b73c45a7ead7f9c4529f8369ed5bfcaa30cac/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b4d5cfa168e74d08f8ba6d58a7e49bc7daef4d58951ec69b66b0d56f4927a68d", size = 3427031, upload-time = "2025-11-05T19:06:51.829Z" }, + { url = "https://files.pythonhosted.org/packages/25/3f/1a192b93bb47c6b44cd98ba8cc1d3d2a9308f1bb700c3017e6352da11bda/openai_harmony-0.0.8-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c007d277218a50db8839e599ed78e0fffe5130f614c3f6d93ae257f282071a29", size = 2953260, upload-time = "2025-11-05T19:06:55.406Z" }, + { url = "https://files.pythonhosted.org/packages/5b/f8/93b582cad3531797c3db7c2db5400fd841538ccddfd9f5e3df61be99a630/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8565d4f5a0638da1bffde29832ed63c9e695c558611053add3b2dc0b56c92dbc", size = 3127044, upload-time = "2025-11-05T19:06:59.553Z" }, + { url = "https://files.pythonhosted.org/packages/1d/10/4327dbf87f75ae813405fd9a9b4a5cde63d506ffed0a096a440a4cabd89c/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:cbaa3bda75ef0d8836e1f8cc84af62f971b1d756d740efc95c38c3e04c0bfde2", size = 2932931, upload-time = "2025-11-05T19:07:01.437Z" }, + { url = "https://files.pythonhosted.org/packages/8a/c8/1774eec4f6f360ef57618fb8f52e3d3af245b2491bd0297513aa09eec04b/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:772922a9bd24e133950fad71eb1550836f415a88e8c77870e12d0c3bd688ddc2", size = 2996140, upload-time = "2025-11-05T19:07:03.438Z" }, + { url = "https://files.pythonhosted.org/packages/60/c3/3d1e01e2dba517a91760e4a03e4f20ffc75039a6fe584d0e6f9b5c78fd15/openai_harmony-0.0.8-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:007b0476a1f331f8130783f901f1da6f5a7057af1a4891f1b6a31dec364189b5", size = 3205080, upload-time = "2025-11-05T19:07:05.078Z" }, +] + +[[package]] +name = "opencv-python-headless" +version = "5.0.0.93" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1d/99/76b7c80252aa83c1af16393454aafd125a0287101afe8deb0a6821af0e30/opencv_python_headless-5.0.0.93.tar.gz", hash = "sha256:b82f9831daab90b725c7c1ee1b36cb5732c367096ac76d119e64e14eb70d5f3c", size = 81817738, upload-time = "2026-07-02T07:01:06.039Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/78/afca939f40ffe2b2380bfa86f812b2f7d4acc5a27b27dc41b49cad7ce7b4/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:10818d91510e05c04568ae12b5cd120779c70c01bf897b001a6221fe430df80f", size = 36521085, upload-time = "2026-07-02T06:55:24.429Z" }, + { url = "https://files.pythonhosted.org/packages/2b/97/8170e9819764c47e436c130d3ff6cfb73b58f923eae9d3a03d8982b04aec/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09a872a157c1376ab922a69bbf22f9a95bcc7b658a9d8b436a60212b02b2eeb4", size = 56563598, upload-time = "2026-07-02T06:55:47.355Z" }, + { url = "https://files.pythonhosted.org/packages/3a/98/1a28a7101e31801042b3098871a74b76c61581d328ef40774ff4edb53a56/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:840bd717c21e5c11cadadc022a823315ea417f961213d06b4df010e019eb16f4", size = 39648433, upload-time = "2026-07-02T06:56:04.255Z" }, + { url = "https://files.pythonhosted.org/packages/9b/21/f6ef335f6e65724aa78b8d792b48d40a48c381715f1e62f5a5049e09d07e/opencv_python_headless-5.0.0.93-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ed709fdf9aa0bd1f2ed8549e71d19449b03a675bb581eb292285f6861953be37", size = 61204038, upload-time = "2026-07-02T06:56:41.823Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.43.0" @@ -2683,6 +4102,67 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/83/6dba32b85f31868400440dc7ad2ca1eab94cbbf3a7b0459ed39f8311a9e2/opentelemetry_api-1.43.0-py3-none-any.whl", hash = "sha256:20acf45e9b21851926835292e4045d290acade1edd2ff3de86d2f069687ba1fd", size = 61912, upload-time = "2026-06-24T15:19:35.434Z" }, ] +[[package]] +name = "opentelemetry-exporter-otlp" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-exporter-otlp-proto-grpc" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/47/b77366bcbe719373a8cac2b4e8ad01f8ff3c9f2c223374d77ece280aae6f/opentelemetry_exporter_otlp-1.43.0.tar.gz", hash = "sha256:65aded6c50ee7dd2b9948c9d0e59ddb4ed4eea6e8532fba95cbe6a4a64a566ba", size = 6086, upload-time = "2026-06-24T15:19:58.003Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/26/d28cc854d2eb779f91351216c51ed0d54886f89f2dd56db9d493ba9fd429/opentelemetry_exporter_otlp-1.43.0-py3-none-any.whl", hash = "sha256:70f3fe740a64596d4157588a2ee7e4fd37d2acc0c0f522a2882b8c29316cd0f0", size = 6726, upload-time = "2026-06-24T15:19:38.437Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/c1/e8098490ab15abf116dcaf9fa89ededcb35547c7d08d4b5a62f573dc1e63/opentelemetry_exporter_otlp_proto_common-1.43.0.tar.gz", hash = "sha256:c4e32ba6d6b13bdb2b8f6764c4fd28d00192826561aa04f6d14eedfce7ac076f", size = 20197, upload-time = "2026-06-24T15:20:00.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/b2/41ebc74ae1d5859901f1b69305de58724bf043381103d6ef413521cbc35a/opentelemetry_exporter_otlp_proto_common-1.43.0-py3-none-any.whl", hash = "sha256:123c3f9cc87218562490c63b36f497bf3a722faf174a515d1443f31ababa6264", size = 17048, upload-time = "2026-06-24T15:19:41.264Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-grpc" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "grpcio" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e2/1d/6336453716ca0a240d4417d19e6d5b77a5e7163e5670ec4f7ec4d3ede7bf/opentelemetry_exporter_otlp_proto_grpc-1.43.0.tar.gz", hash = "sha256:1b3e0627daa9bc21884d4a13946807c255eb558bfe5bdd543dffb6f4c9faee0d", size = 27213, upload-time = "2026-06-24T15:20:00.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/74/2700b5d5c946bf2dba87073fce3dfc198c46bc92ea3d5693f54bc51c90b1/opentelemetry_exporter_otlp_proto_grpc-1.43.0-py3-none-any.whl", hash = "sha256:6a10d1feacffffda19acacbf277b736094b1e2f4dbb98c90ccb2c6e1962e2ec6", size = 19626, upload-time = "2026-06-24T15:19:42.233Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/92/0b9f56412483a8891d4843890294796c9df8ab42417bd9bad8035d840cb3/opentelemetry_exporter_otlp_proto_http-1.43.0.tar.gz", hash = "sha256:fa8a42bb7d00ee5391f4c0b04d8e6a46c03caa437903296ab73a81dc11ba118f", size = 25406, upload-time = "2026-06-24T15:20:01.515Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/20/b685ed7af2e17c29ffc8af56f1fa8bc2033258fc30fb0d2b722f49d13ba0/opentelemetry_exporter_otlp_proto_http-1.43.0-py3-none-any.whl", hash = "sha256:647f603aa8efdbdb4dbff842e0729d0406a6fff26b295a72d3d60e7d963b2610", size = 21795, upload-time = "2026-06-24T15:19:43.164Z" }, +] + [[package]] name = "opentelemetry-exporter-prometheus" version = "0.64b0" @@ -2692,9 +4172,21 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "prometheus-client" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/97/b3/f778f705289ea5f1c54589ae4494d318c9cede052f18ca33979fda0ef016/opentelemetry_exporter_prometheus-0.64b0.tar.gz", hash = "sha256:96fec79be9527cb9dc994d7e663051df35161eb936fe2d41954725e4595abbc1", size = 16405, upload-time = "2026-06-24T15:20:02.278Z" } +sdist = { url = "https://files.pythonhosted.org/packages/97/b3/f778f705289ea5f1c54589ae4494d318c9cede052f18ca33979fda0ef016/opentelemetry_exporter_prometheus-0.64b0.tar.gz", hash = "sha256:96fec79be9527cb9dc994d7e663051df35161eb936fe2d41954725e4595abbc1", size = 16405, upload-time = "2026-06-24T15:20:02.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/dd/d95db7ac5bec037b3dcc8ea6fe64cd5dc4bc87017d2e06e7410318e85dc0/opentelemetry_exporter_prometheus-0.64b0-py3-none-any.whl", hash = "sha256:9979a15f8d007d442bc7a6e16f4cbde5e8e0c5e99689887ceb5f33da251f0655", size = 13031, upload-time = "2026-06-24T15:19:44.159Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.43.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e0/b9/d357faefb40bda1d4799913e6af611171ff22a2dedcb93576bc92242d056/opentelemetry_proto-1.43.0.tar.gz", hash = "sha256:224778df17e1f3fafeaaa21d874236ca5f6ffc2f86e0899298ec7351aac27924", size = 46481, upload-time = "2026-06-24T15:20:07.625Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/dd/d95db7ac5bec037b3dcc8ea6fe64cd5dc4bc87017d2e06e7410318e85dc0/opentelemetry_exporter_prometheus-0.64b0-py3-none-any.whl", hash = "sha256:9979a15f8d007d442bc7a6e16f4cbde5e8e0c5e99689887ceb5f33da251f0655", size = 13031, upload-time = "2026-06-24T15:19:44.159Z" }, + { url = "https://files.pythonhosted.org/packages/ed/a7/3e5308cf548b8f72529c7db1afdb3a404211982376a12927fd7759f77bf3/opentelemetry_proto-1.43.0-py3-none-any.whl", hash = "sha256:c58f1f7ef84bc7dc2834016c0c37fe0081dde7ca9f6339be1970fbf9cdaaa90d", size = 72489, upload-time = "2026-06-24T15:19:51.164Z" }, ] [[package]] @@ -2724,6 +4216,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/ca/23ba87a221b574a7c5a99d48849d80bfe8b047624681357e2b002e566187/opentelemetry_semantic_conventions-0.64b0-py3-none-any.whl", hash = "sha256:ea77e85e354b8f604ddbe5f3d9135216f982fa4d77e5859ac30f6d8a50505aa6", size = 203713, upload-time = "2026-06-24T15:19:53.339Z" }, ] +[[package]] +name = "opentelemetry-semantic-conventions-ai" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-sdk" }, + { name = "opentelemetry-semantic-conventions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/24/02/10aeacc37a38a3a8fa16ff67bec1ae3bf882539f6f9efb0f70acf802ca2d/opentelemetry_semantic_conventions_ai-0.5.1.tar.gz", hash = "sha256:153906200d8c1d2f8e09bd78dbef526916023de85ac3dab35912bfafb69ff04c", size = 26533, upload-time = "2026-03-26T14:20:38.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/22/41fb05f1dc5fda2c468e05a41814c20859016c85117b66c8a257cae814f6/opentelemetry_semantic_conventions_ai-0.5.1-py3-none-any.whl", hash = "sha256:25aeb22bd261543b4898a73824026d96770e5351209c7d07a0b1314762b1f6e4", size = 11250, upload-time = "2026-03-26T14:20:37.108Z" }, +] + +[[package]] +name = "outlines-core" +version = "0.2.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/04/4a0812eb27c086cfd2e66e7ec9150f33e105912a9b7f8b335e3479f03a06/outlines_core-0.2.14.tar.gz", hash = "sha256:64808deed1591ca3029ff64346ceb974cd5d780c916ea82504951fe83523039e", size = 191539, upload-time = "2026-01-09T15:59:10.016Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/69/e0be45d4c8ad7d301cdc9917d22ff39211da1e830f92fb07b29c9221b5c4/outlines_core-0.2.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:615566bf8257d2bba8ac192cdfc29d1c4357f57b53672fbd622e821215e4f1bd", size = 2338968, upload-time = "2026-01-09T15:58:23.317Z" }, + { url = "https://files.pythonhosted.org/packages/f2/67/9dab90313460eb250f926e7985d62cebfc33c7580197be8a496de6e9f7c4/outlines_core-0.2.14-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:81d01cfae29de5671bc5013fd6b2008621157bec3d8be284da7da2dc0672745c", size = 2236169, upload-time = "2026-01-09T15:58:24.575Z" }, + { url = "https://files.pythonhosted.org/packages/29/29/3a04944407207a5d214879ca5ca33c2bd3e65199a4e927051c1bdaaa4d50/outlines_core-0.2.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3bb2060c240c4507f334965a8948dbeeb22007560d797f6debd92346c0b620cb", size = 2341426, upload-time = "2026-01-09T15:58:33.553Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a7/a77f746272504bac3f628047d56ea1731b61549a3e1d9bbfd226f2968246/outlines_core-0.2.14-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1de34681c7e0e7e1551fc9036e4fa3c57986336c905a10536591ceb6d869c258", size = 2236941, upload-time = "2026-01-09T15:58:35.118Z" }, + { url = "https://files.pythonhosted.org/packages/c1/9a/4b62903de006d991b58674ff033c1b6fb92be5767360376fc961f6771bdb/outlines_core-0.2.14-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6453e23f01d98ec48e3a4141d7112792ce77001dfb28d91d6fd89f47009f91ef", size = 2341051, upload-time = "2026-01-09T15:58:44.415Z" }, + { url = "https://files.pythonhosted.org/packages/50/36/1532f7d9ab16c676812d94528e89964aa0d15f12adcb285e6ed86f86f2fe/outlines_core-0.2.14-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7deef6df74cb247f2a3a62f03438ba967456504b0555ec7029f8db834e054448", size = 2236778, upload-time = "2026-01-09T15:58:45.437Z" }, + { url = "https://files.pythonhosted.org/packages/d5/63/dfa000239e46f17b47e6dc9bec3aab8a8136fe400312f1916320e02c8f38/outlines_core-0.2.14-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1776ae984574461f249fe590314a439992eb9b883f4091b8fa7fc56f29f3717", size = 2343210, upload-time = "2026-01-09T15:58:54.282Z" }, + { url = "https://files.pythonhosted.org/packages/36/4f/0e63da06c6054f154ef22b5ef3c6b9030cb22da9c03d2d2dd82836a1e795/outlines_core-0.2.14-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:7eba2b41dac03d6e6e8d5ea0aecbbc03dacb4c57de3b1fc944d0bafb022941f7", size = 2238206, upload-time = "2026-01-09T15:58:55.705Z" }, +] + [[package]] name = "overrides" version = "7.7.0" @@ -2756,7 +4277,8 @@ name = "pandas" version = "2.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "python-dateutil" }, { name = "pytz" }, { name = "tzdata" }, @@ -2823,6 +4345,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b6/61/fae042894f4296ec49e3f193aff5d7c18440da9e48102c3315e1bc4519a7/parso-0.8.6-py2.py3-none-any.whl", hash = "sha256:2c549f800b70a5c4952197248825584cb00f033b29c692671d3bf08bf380baff", size = 106894, upload-time = "2026-02-09T15:45:21.391Z" }, ] +[[package]] +name = "partial-json-parser" +version = "0.2.1.1.post7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/6d/eed37d7ebc1e0bcd27b831c0cf1fe94881934316187c4b30d23f29ea0bd4/partial_json_parser-0.2.1.1.post7.tar.gz", hash = "sha256:86590e1ba6bcb6739a2dfc17d2323f028cb5884f4c6ce23db376999132c9a922", size = 10296, upload-time = "2025-11-17T07:27:41.202Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/32/658973117bf0fd82a24abbfb94fe73a5e86216e49342985e10acce54775a/partial_json_parser-0.2.1.1.post7-py3-none-any.whl", hash = "sha256:145119e5eabcf80cbb13844a6b50a85c68bf99d376f8ed771e2a3c3b03e653ae", size = 10877, upload-time = "2025-11-17T07:27:40.457Z" }, +] + [[package]] name = "pathspec" version = "1.0.4" @@ -2972,6 +4503,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/74/c3/24a2f845e3917201628ecaba4f18bab4d18a337834c1df2a159ee9d22a42/prometheus_client-0.24.1-py3-none-any.whl", hash = "sha256:150db128af71a5c2482b36e588fc8a6b95e498750da4b17065947c16070f4055", size = 64057, upload-time = "2026-01-14T15:26:24.42Z" }, ] +[[package]] +name = "prometheus-fastapi-instrumentator" +version = "8.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "prometheus-client" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/f4/cdcebf7094b03b99fba71ac8f56bd6f227973642662f49d272332d8419b3/prometheus_fastapi_instrumentator-8.1.0.tar.gz", hash = "sha256:b77f3043665e8d28e2bbd21017506195a43d9adf1d402d01bf95b494b7e560e1", size = 20492, upload-time = "2026-07-26T11:12:44.202Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/b9/91a2246e6cf01b7ccb14479803c8a50f9c258ae5c6a0f16ba3294820632b/prometheus_fastapi_instrumentator-8.1.0-py3-none-any.whl", hash = "sha256:b9f40b2cff3f7891ca0610b3ae4fc6ec723fd326b04bb659819aaeb821a0fc7d", size = 19649, upload-time = "2026-07-26T11:12:45.168Z" }, +] + [[package]] name = "prompt-toolkit" version = "3.0.52" @@ -3109,17 +4653,17 @@ wheels = [ [[package]] name = "protobuf" -version = "7.35.1" +version = "6.33.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/01/9ef0afd7999eb9badb3a768b4aedd78c86d4c65cfaf1958ab276199e76b4/protobuf-7.35.1.tar.gz", hash = "sha256:ce115a26fe0c39a2c29973d914d327e516a6455464489fe3cd1e51a1b354f81a", size = 458717, upload-time = "2026-06-11T21:55:40.257Z" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/10/03/8aeeb7458d22546bf64b5250ca1daeb5ff757d900e8e4a7476c6f0db843e/protobuf-7.35.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:24f857477359a85c0c235261b8ba905fd51b2562f4a64ca1df5473f29850cbf6", size = 433226, upload-time = "2026-06-11T21:55:31.719Z" }, - { url = "https://files.pythonhosted.org/packages/37/4b/dfb89eb0e652a1ff073c39a59fb5e3a83cfe9b57a2c83fa6d78270101767/protobuf-7.35.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:11d6b0ec246892d85215b0a13ca6e0233cf5284b68f0ac02646427f4ff88a799", size = 328847, upload-time = "2026-06-11T21:55:34.035Z" }, - { url = "https://files.pythonhosted.org/packages/0f/58/dc12f2cd484951524af6e3382c785869b9b3fb5e52ee95ae23add53ee8f9/protobuf-7.35.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:b73f9489a4b8b1c9cb1f8ed951c736392592edb24b9d6819f36d2e10b171d5b4", size = 344030, upload-time = "2026-06-11T21:55:34.941Z" }, - { url = "https://files.pythonhosted.org/packages/e4/be/5b3cfe508bfab6761414ff944e3366eb13be4fd71efcd69450f89ba39f43/protobuf-7.35.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:74758715c53d7158fb76caf4f0cfdacc5329a4b1bb994f865d6cf302d413a1c4", size = 327130, upload-time = "2026-06-11T21:55:35.921Z" }, - { url = "https://files.pythonhosted.org/packages/d8/bc/6d6c7ba8709c85f8f2c390b2b118d6fb08a783676a572271851bf45a7d22/protobuf-7.35.1-cp310-abi3-win32.whl", hash = "sha256:353652e4efd0bca5b5fc2656abf8307ef351f0cf938c9eba09f0e09c20a25c30", size = 428945, upload-time = "2026-06-11T21:55:37.034Z" }, - { url = "https://files.pythonhosted.org/packages/0a/19/8d0cb6f20a1ef7b18f1c8986ad5783f22f84cce39c6ce9a6e645ea55192e/protobuf-7.35.1-cp310-abi3-win_amd64.whl", hash = "sha256:230a75ddfc2de4806e56696ce9640c1cdfdb6543b7cfce98d42a4c0a0e7bdb87", size = 439996, upload-time = "2026-06-11T21:55:38.123Z" }, - { url = "https://files.pythonhosted.org/packages/19/c7/5f7c636ec43e0c545e28d1f1db71990108306f7bdcb89f069ba97e428e7f/protobuf-7.35.1-py3-none-any.whl", hash = "sha256:4bc97768d8fe4ad6743c8a19403e314511ed9f6d13205b687e52421c023ac1b9", size = 171659, upload-time = "2026-06-11T21:55:39.155Z" }, + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, ] [[package]] @@ -3168,6 +4712,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842, upload-time = "2024-07-21T12:58:20.04Z" }, ] +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + [[package]] name = "pyarrow" version = "24.0.0" @@ -3218,6 +4771,138 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, ] +[[package]] +name = "pybase64" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/65/c513eab7211590250f729a06aacc0bc95eaf760b9235666e933d200105d0/pybase64-1.5.0.tar.gz", hash = "sha256:545ab2a433769e3b8e1ce2b4f7b07218bbde202f4954fbfe52948b2522120727", size = 149492, upload-time = "2026-08-08T15:42:00.205Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/f8/96c1413310f696ecd3364d887073f73708121469ff7f3bd3e24ec8dece6d/pybase64-1.5.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:045fa2f3f5da6cfa86822c645b92e18cfc7c13babccb5ceec9bb64a17ac3f1bc", size = 90662, upload-time = "2026-08-08T15:37:57.091Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ee/591f24aef04e1ca569450b85c86df0f8a8b3dda04f68cdbf6101312456d7/pybase64-1.5.0-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:93bc9bdfaf87dc7d79ee0182b255383b7f82a3167d0166b99330d897b59f9053", size = 94201, upload-time = "2026-08-08T15:37:58.696Z" }, + { url = "https://files.pythonhosted.org/packages/20/c0/66a721f8d0d5f3d43704b78b30ddc51d07eef24ddf94470c8e1808b0826f/pybase64-1.5.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8b08e4a065c9fa88ab9b8a2345b58073776806488b1ff5e4348957d0aa218043", size = 83843, upload-time = "2026-08-08T15:37:59.817Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cf/44aa61b4738b9827e50fc081c9072d553fc538a372580a6326771848d1cd/pybase64-1.5.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:897ca382ec6c7bad041ce7b3a64b3a15f1b639dfea89ffcf29bdd235c706fac3", size = 79555, upload-time = "2026-08-08T15:38:00.942Z" }, + { url = "https://files.pythonhosted.org/packages/98/f2/db862e347968eeecf96729244f092a8e1d9bbc1daf94aef4baf3446296d5/pybase64-1.5.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3398eb35a82a94d61756f7a4ad6a1c5a3e735c6abb97167398a22389a9b8ca7a", size = 81765, upload-time = "2026-08-08T15:38:02.206Z" }, + { url = "https://files.pythonhosted.org/packages/86/37/2ec5e90db7c1d01c126b02933adb31838ed8f4d8834193c4f2c1440e2c28/pybase64-1.5.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:c3935b4402f257d9c7448944db07f91d6fc20453f8c3f0fa1bf26c490b534c84", size = 80619, upload-time = "2026-08-08T15:38:03.408Z" }, + { url = "https://files.pythonhosted.org/packages/9d/80/09093682d7834a0cf8516b4d9b2b9ba579abb54c56046666ab12f97208d7/pybase64-1.5.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:a167f17421c237a32c93072a053ff756d9fb225e69a620c3f4818665f0520044", size = 78240, upload-time = "2026-08-08T15:38:04.484Z" }, + { url = "https://files.pythonhosted.org/packages/20/6c/57d95e6ff206d7aabb0a6ec54a8d860ca1e0c84fee611eacfe95945b7478/pybase64-1.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:716aed288780c9c2081943a3a7b5be6993cdad56b0cdcb4ef4b562ef56c5a1ae", size = 81888, upload-time = "2026-08-08T15:38:05.558Z" }, + { url = "https://files.pythonhosted.org/packages/77/71/d0559d16467c42ac91501e5e423d65be1801dd5ebf40f4a244134c760a46/pybase64-1.5.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:d373b682dd0a267ece21869ca9a48d40b55120a3be714661ad0e9afdce9ce27e", size = 75233, upload-time = "2026-08-08T15:38:06.645Z" }, + { url = "https://files.pythonhosted.org/packages/40/2e/4dfee2d5c37473cb91204dac4f1df83710da30fcd93ae345c2a5acb6bdb9/pybase64-1.5.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5d02948944dad3e99ebe70a3049d7df66f5faba97ed03b411349b034558ed936", size = 90524, upload-time = "2026-08-08T15:38:07.791Z" }, + { url = "https://files.pythonhosted.org/packages/8d/07/e2595a9b32d2e635514de93847e1dca2bd6361c34b2e142b46b0eac852f0/pybase64-1.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:d83f517403ff39404b8586d07e97c019cb2a7cb6665cb070c6aebf1fc03e5487", size = 79217, upload-time = "2026-08-08T15:38:09.004Z" }, + { url = "https://files.pythonhosted.org/packages/13/ce/57bc5269db8cd07f427e7b42149f00194106c80d02f4147411005dee7522/pybase64-1.5.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:216b78caa73ae9b82f3f006e9694ee5a1bde89e50f4552fd1679b56b080cfb7e", size = 77807, upload-time = "2026-08-08T15:38:10.195Z" }, + { url = "https://files.pythonhosted.org/packages/bd/08/f366686f58858af2ec5dcedb80c394bf264927ba12c99d9a7233cca16c66/pybase64-1.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0855f67fa47c0bdf237ea875c11afce2a8cd879644b288d3f05ed9effab17953", size = 78181, upload-time = "2026-08-08T15:38:11.321Z" }, + { url = "https://files.pythonhosted.org/packages/1d/fe/de44b16a234b12adf2a2b26a758805de8e7a88e2ccd547053d3dd96d8c57/pybase64-1.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a707d36935229ae5c3044cd601908cb7bd9f25757003d029765ccf66818301ce", size = 92950, upload-time = "2026-08-08T15:38:12.464Z" }, + { url = "https://files.pythonhosted.org/packages/1d/66/9f1be6a4db86577eebf3106496a2a791b37e5fb74695d4c8eeedbd04490a/pybase64-1.5.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:80b171f1546935be4dae1e01bfd8630d2712271e067858b7135726e7d9bc7cce", size = 91058, upload-time = "2026-08-08T15:38:18.983Z" }, + { url = "https://files.pythonhosted.org/packages/af/36/4e44a0688efe26434bf378b4565b01ac94f81422e8a5746291a03472cd56/pybase64-1.5.0-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1a2b9cf39b4d30f600df8c56cccbc03adfc6e1ae8c04cd6b181105a432d4a515", size = 94681, upload-time = "2026-08-08T15:38:20.59Z" }, + { url = "https://files.pythonhosted.org/packages/aa/d1/fc02005906fd48081b7b8f077cd422a55399fa351c2a6d3e5fed951794ce/pybase64-1.5.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:865b7db127a95e33640ebcdb4bb3aad165d4873ee7c1008949129f3c4f900dd8", size = 84634, upload-time = "2026-08-08T15:38:21.711Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c6/5bb0f21a9f4d231339a42f16ebabc7c6d9a7d619e756327b15a474650ece/pybase64-1.5.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:3344ce336d9d8292125369c1475d1663e7e1a06894e8e5150307e11f782c6afd", size = 80455, upload-time = "2026-08-08T15:38:23.05Z" }, + { url = "https://files.pythonhosted.org/packages/b8/04/0ba9a1f2ea39baf081dd44d22d710d9b050ce15991d641982f1814508484/pybase64-1.5.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1aaae81669bf18b5a35dcb43dbb200f52b13f847a56bed7a2e82f31cc6f9f74d", size = 82304, upload-time = "2026-08-08T15:38:24.156Z" }, + { url = "https://files.pythonhosted.org/packages/c1/9e/6b380ff964dd77b79cc1ce565b73780345132e0e181d315f31a2263c5e1f/pybase64-1.5.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fb5dc922ce3cb4211caa7e29e6daee98f319e59f297a904acd74f2fdd0674356", size = 81259, upload-time = "2026-08-08T15:38:25.327Z" }, + { url = "https://files.pythonhosted.org/packages/b9/93/dd7fd7f8ed228f7735ec59a9f85f3c683cef371a76b29520344655bf7c97/pybase64-1.5.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:356e7bd1453551c06231df8411bfbaed9998fbcba2da723d84fb270ff1f977a7", size = 78360, upload-time = "2026-08-08T15:38:26.678Z" }, + { url = "https://files.pythonhosted.org/packages/d8/99/b5e9e7d4b5e49d7a984c4a26b48bdf988ec62c2778df80144af1a39bd4b1/pybase64-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:11dfa286f6c5fe6795430bf08fc44b64c98e208558215b0590c9f28fd99a92e3", size = 82358, upload-time = "2026-08-08T15:38:27.856Z" }, + { url = "https://files.pythonhosted.org/packages/67/fa/19d11ee70fbdb10e574a39ad7fc7adc06e5635a2b2ac291a6554c7c651ae/pybase64-1.5.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6be40c3311eabe8a816e00041844f9b249828015dc98be8a48a7c3275954ee76", size = 76384, upload-time = "2026-08-08T15:38:29.169Z" }, + { url = "https://files.pythonhosted.org/packages/71/32/a83622dfa3162dd6fcd019dd8fbb766f0ce064fe67b3d3d2759881dbac4e/pybase64-1.5.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:4e8b163c8d2d2a5f414f2c31cdd91024e0c91c72e735a9a564a62460ac838acb", size = 91407, upload-time = "2026-08-08T15:38:30.306Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b5/1707748813784af0b1340f6c6525887f1ecb393c3f88070a2bb2d86bd94e/pybase64-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0030a64fe91791e5e553edaff3a55d319cd07fb5e097b09c5f7f45e4905c40cb", size = 79687, upload-time = "2026-08-08T15:38:31.771Z" }, + { url = "https://files.pythonhosted.org/packages/e9/ee/8101e43b5cc070c0adf298f87500154c13b9097d4456a2c1aadd71339329/pybase64-1.5.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:28d5db510433bb1544dc128c4e7ebd85ae57cec2a4608edd1f7ca4fed3e53b3d", size = 77913, upload-time = "2026-08-08T15:38:32.898Z" }, + { url = "https://files.pythonhosted.org/packages/70/8c/43b2281077ca9a531bd896b7a9fe871d091d80d172d68e439c7aa6337033/pybase64-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:26422429a0bb2f15773dacc0fcb1bcddfce68c6b2d41fc14bc7fc17f8c529542", size = 79172, upload-time = "2026-08-08T15:38:33.974Z" }, + { url = "https://files.pythonhosted.org/packages/c2/1a/b536e571518eb2f4a2db1c6c7c5913af5780ff82c9eefb41f674fed71ceb/pybase64-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ccbae849677648be456ea0de769a78e432d2d24f71cbdc739741e69f8160e0d7", size = 93636, upload-time = "2026-08-08T15:38:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/a5/17/a1fc8e55551530876d3be31079b8701b7f5ac8451b63a08a19a4f9714454/pybase64-1.5.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:75d21d0a2cae0bb071c68686d77e5100be611ec4e80e0d97f8736c27da0ab197", size = 39681, upload-time = "2026-08-08T15:38:42.145Z" }, + { url = "https://files.pythonhosted.org/packages/9d/2c/b46f7e0c1ea482db0f8445d5bfad7e5a4f39d977868e10b4c3823e94fa20/pybase64-1.5.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:1bde27266ec4a56c38ef8e17998e430d30cc6310fde76332381bf5aaa81872ba", size = 40200, upload-time = "2026-08-08T15:38:43.354Z" }, + { url = "https://files.pythonhosted.org/packages/da/12/085dc70e757e6101c8f61239bae538640aac60ddfebb41e2534af3712e14/pybase64-1.5.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:220d8ab003d44144d80f8b776019adedc23fdc7bcb270396744b9805a8186d0e", size = 46726, upload-time = "2026-08-08T15:38:44.378Z" }, + { url = "https://files.pythonhosted.org/packages/e5/c3/8171fd18a57218c5e7c252f658709f9bd3d0eece9d4196542230103a53d6/pybase64-1.5.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:a1529b8e08a93dd9c00d1e3b3c2b627a9600d96c2f40143dc0b3a85f48fa85e5", size = 92183, upload-time = "2026-08-08T15:38:48.038Z" }, + { url = "https://files.pythonhosted.org/packages/23/84/b91aabd22a65a3679633855dde720dfb86571e15f88a9b1b295adda90e8c/pybase64-1.5.0-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0be37689b624ae293394fc826c9a048c6118520d6a962de033ffb054564bf61f", size = 95718, upload-time = "2026-08-08T15:38:49.104Z" }, + { url = "https://files.pythonhosted.org/packages/67/cd/441fd3b9bc7a49846362fb52a0971cee6da4dca2eb8545100ec043b2a0da/pybase64-1.5.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bf98b77c6cca5c5da30135b69b30668da07a32d41210c62121b34c84239d9d4a", size = 86068, upload-time = "2026-08-08T15:38:50.683Z" }, + { url = "https://files.pythonhosted.org/packages/2f/24/48cfe7e1b776c0af1ce5240f7e71383890cd361242e537b6c510804a68d2/pybase64-1.5.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:0578c54f1ae89e6175eddb742dbaf2e95a060735ec11f4b661f762b635680cbd", size = 81077, upload-time = "2026-08-08T15:38:51.825Z" }, + { url = "https://files.pythonhosted.org/packages/db/38/5b47895e2f19f9775a3daaec98a652ba7c0ccfb480c223d981c2ec75c0ed/pybase64-1.5.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:ae78cdaec57f21e7f44cc5f9866d694cc072e1b1082286f30fd74e7545fa2916", size = 83387, upload-time = "2026-08-08T15:38:52.921Z" }, + { url = "https://files.pythonhosted.org/packages/74/2d/115526e63080e96ce039619a1a29a4fe49d138c5d7d525b6adbccf0c1c0f/pybase64-1.5.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:1f315f07b269f074995c445b65dfde62d12c0e889e9c3b0534befdb05866e880", size = 82460, upload-time = "2026-08-08T15:38:54.436Z" }, + { url = "https://files.pythonhosted.org/packages/53/b8/8970ecca7a5945f81d34f9a91d23169f7e62e2487ef3694e0004943e7243/pybase64-1.5.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:99570e43605b9c849ff1606e1691e503962250f80ec3e827249f7ad820e402d8", size = 79359, upload-time = "2026-08-08T15:38:55.69Z" }, + { url = "https://files.pythonhosted.org/packages/55/06/eea9cb5955430d5f789c18eab854284c66b1a024efae4928992d44bcde65/pybase64-1.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e0143b3515b97bb3c4743fbdf10f53950c0bb1fe1a2db1054b422ba370594333", size = 83768, upload-time = "2026-08-08T15:38:56.793Z" }, + { url = "https://files.pythonhosted.org/packages/de/2f/a121c58260d63d16861fd936373d07c4ab0cef51b0d7391cafaf8e4648c0/pybase64-1.5.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:b0597ca31c472f3071844648ce5ab86a1732033ca230daffd8f87c6f8596a8ae", size = 77416, upload-time = "2026-08-08T15:38:57.995Z" }, + { url = "https://files.pythonhosted.org/packages/24/6a/ea3a1078de626ce765402d6d3e1cb6d69f83104646bcf2e2772983be77aa/pybase64-1.5.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8d303baddeddaccada149bbee270b3e2eedcaec2df082834895cdd897a602674", size = 92473, upload-time = "2026-08-08T15:38:59.149Z" }, + { url = "https://files.pythonhosted.org/packages/8d/9c/22878279f1663bea15b5211056e3c8cb19c4783d2566a0032bcfa37d678b/pybase64-1.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:a34261348f88443d9e234f251a1f1fcb711c1cc006824fdb29b649735d8ac35f", size = 80804, upload-time = "2026-08-08T15:39:00.271Z" }, + { url = "https://files.pythonhosted.org/packages/9b/f0/57c36867282341ccc47c0db67590dd8f0c621fd435aa5944bec4713138b5/pybase64-1.5.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e675b15b7a7b81e5b1a1e747cc49f9f9e6649d3b5e8a61719b46b9a671433210", size = 78871, upload-time = "2026-08-08T15:39:01.429Z" }, + { url = "https://files.pythonhosted.org/packages/4d/ce/23b80fde747156f6387a2f769fac1384e2e34cd4f07daa32e990991eb64a/pybase64-1.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:a1f8f1bb4158069291fe6ac2d34db942418f2804564d04b8e97722041035f843", size = 80451, upload-time = "2026-08-08T15:39:02.764Z" }, + { url = "https://files.pythonhosted.org/packages/bf/02/1486ad47fc065bbaa45c12229673bb03f0480dabdba408b04a54ac480264/pybase64-1.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0abc0f2312c17765bf92dd382982cca9dc1b0148bf0d708f5f88339d84bb7687", size = 94725, upload-time = "2026-08-08T15:39:03.877Z" }, + { url = "https://files.pythonhosted.org/packages/43/ec/bf6a0df18b4a627a2ad3c8897e67797cb8128fed8cda2b654dd9ddebba25/pybase64-1.5.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:92998479a2a4464d141ef709e52dc3e4d4d4ce7f3b9cb5052d2c56c55b405b15", size = 33074, upload-time = "2026-08-08T15:39:04.939Z" }, + { url = "https://files.pythonhosted.org/packages/92/41/cef45112b1c853c58a5a47dc4fb823d1cd7c79cf24bb8424ef7fd3fbb180/pybase64-1.5.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:2b5563aca0b7b74751dafe6cc3e1850a3401414c05342f1bbeb26549b5c3bda0", size = 39687, upload-time = "2026-08-08T15:39:12.267Z" }, + { url = "https://files.pythonhosted.org/packages/7c/fb/9058281862be3a2a12b1b2bd48addf8e0eaa085c1cf75e22d49663b22a9a/pybase64-1.5.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:b6cb9e548816e0838b10d29b061cfbbfc81b726f6c5f89d60e83bd7d703ed06b", size = 40208, upload-time = "2026-08-08T15:39:13.588Z" }, + { url = "https://files.pythonhosted.org/packages/60/f0/f6ff0e564d4d2f4ac9161d6a8445cbfb317c83ad9f79deca3c3bf27b8b79/pybase64-1.5.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:435064ff2fc778a02d1234289a22050a4d3b29752062b5ecaf45eae62273ec47", size = 46726, upload-time = "2026-08-08T15:39:14.689Z" }, + { url = "https://files.pythonhosted.org/packages/6c/58/cce345652b019e2a80e51b8d31bd4fa1662612ff1260dfedbcd5e1675106/pybase64-1.5.0-cp314-cp314-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:fb1734c69974acaee369726b48031c0d0117830bc050188086a69227c32d2426", size = 92385, upload-time = "2026-08-08T15:39:18.169Z" }, + { url = "https://files.pythonhosted.org/packages/86/0f/c332c26d75b0f2bcab549fe746b6978b2928d8b94fe226333c7e94ecfdd1/pybase64-1.5.0-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b391e54bc8198387cf089ffd343d8c99d58e73f209c31aa2e5f420bf20bbb0c7", size = 95720, upload-time = "2026-08-08T15:39:19.355Z" }, + { url = "https://files.pythonhosted.org/packages/f0/e5/0476f7b28544d29225bc1b0be5fd613ab62c38080c65d55299a8f1e7e334/pybase64-1.5.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1626f1de1d7c109e25e20528cf1ffe17d0b614baa87c9d20f6181cb65234168", size = 86226, upload-time = "2026-08-08T15:39:20.473Z" }, + { url = "https://files.pythonhosted.org/packages/75/5d/5664794aff60d8df94371a466171940c3ecb081d76d24ca1327dd32aed60/pybase64-1.5.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:ade98a94cd71692baf0ab21245ecf9a2f1c275460dc4106e23ce9aca1c4c1838", size = 81080, upload-time = "2026-08-08T15:39:21.899Z" }, + { url = "https://files.pythonhosted.org/packages/f3/32/6ce14f3209f1629e11b11f1c44f545b87ebe88a2f35e469526d72f2fe0db/pybase64-1.5.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:cbc41c5376b30ba7b3d558505f7598799034c8aef30e3cee00f32bf8d26fbede", size = 83557, upload-time = "2026-08-08T15:39:23.429Z" }, + { url = "https://files.pythonhosted.org/packages/38/5e/0d73f7f9d3e4579df08af94847e39a675b654b4c99330ef1b5718594406c/pybase64-1.5.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:be98a4e72e3821714770ed290e5e1a8a6cabe77af58520a9adf718acc43a165e", size = 82370, upload-time = "2026-08-08T15:39:24.696Z" }, + { url = "https://files.pythonhosted.org/packages/d9/f4/0831d2736370a4afc690aabfe60e295e2773456efdc764513974f7b2b2d9/pybase64-1.5.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:a8bc9cb80cd736785aa39be5e5d934772a36f9ba30fa71b7c19dbe1da44a306f", size = 79538, upload-time = "2026-08-08T15:39:25.859Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/c897af35bdc3f4e26d1050c8ada1eb91dff87e601681bb6b8a3f47db6b42/pybase64-1.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:aabdeccdd1be80735cd8cb815565d9528c767113358fac2e8eba21030e018a65", size = 83889, upload-time = "2026-08-08T15:39:27.024Z" }, + { url = "https://files.pythonhosted.org/packages/27/44/89be4c77ebbba058ea1d263a62349d306b13d92626b2593c4b56e01321d2/pybase64-1.5.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:9d16bd1cdbb63985cb2f3ec4bda4de13ba6396c1f81468941c650b4157670ee1", size = 77092, upload-time = "2026-08-08T15:39:28.573Z" }, + { url = "https://files.pythonhosted.org/packages/f3/fe/eb0723048975618b73f3dc4b3b4e906b17aacd50916f3de3350a9980fbfc/pybase64-1.5.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:37daeed30664d0d59dc0c99707a3a3fb723b8dffdf62266078308b9b26c7a18f", size = 92793, upload-time = "2026-08-08T15:39:29.739Z" }, + { url = "https://files.pythonhosted.org/packages/2d/c7/a6bb0ac9aa970322ea30b37af8054a8110d2422cbe7bcaf99cee110d77db/pybase64-1.5.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:40fd8e16bfde1e9d80700bbdb51a830c0f7e384c2130c4a8ed5f0912fb269cce", size = 80873, upload-time = "2026-08-08T15:39:31.017Z" }, + { url = "https://files.pythonhosted.org/packages/6c/00/75c2ccacfc7bd47d50bdc91fee3e09582ca9bf047414fcd44ed9d61e55a5/pybase64-1.5.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:1c32f2078df7e3c4f7e573592cdcd8eb50c827cd51226291ee867c217f036abe", size = 79031, upload-time = "2026-08-08T15:39:32.269Z" }, + { url = "https://files.pythonhosted.org/packages/f5/ae/f6cdaea0a266cdeb485f6088551b8413361947f008de21ebe1479b5c5042/pybase64-1.5.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:a119cdd2e59b30aa570e75182b22fa149da50e921ed8b4c492eb9ed308d944c0", size = 80520, upload-time = "2026-08-08T15:39:33.463Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f8/7961ef761d93b1bca865dc84e99ce071f73b05a8f73e2759e19b42732d1f/pybase64-1.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:82b38c11b73d4ea37b1d76d4690131472ce6a144166a63fedf336d88a101336b", size = 94804, upload-time = "2026-08-08T15:39:34.589Z" }, + { url = "https://files.pythonhosted.org/packages/ce/76/4dcf99fb78ed8cbb5c45d4a4580ed7d3206ddf098f4d9bb03f9f292c3e7e/pybase64-1.5.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:6260074fae5bc47838af0fee1a6f48530d1ac7b5f49c80868144ba2f69f43145", size = 33041, upload-time = "2026-08-08T15:39:35.771Z" }, + { url = "https://files.pythonhosted.org/packages/86/69/54f004e0f5ab8e7a96b1a43198e2fb554c2a94c4b78f553ebfef733377b7/pybase64-1.5.0-cp314-cp314t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:0872880c9150fc79347c658937507033b8e600570569e4494e1230987e91be04", size = 97305, upload-time = "2026-08-08T15:39:43.093Z" }, + { url = "https://files.pythonhosted.org/packages/a7/cb/d1d33080136e437e49d73bad2f27a1ac3129b058585c71fdab2c8783fa2c/pybase64-1.5.0-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:106dc1813dff9ad1e936ab6de486bc0e19d281741c1cdcb3effe31602c571d71", size = 101290, upload-time = "2026-08-08T15:39:44.318Z" }, + { url = "https://files.pythonhosted.org/packages/1e/a0/b935f321d7f4e6317487b9452492287f4b709465290445bb8daa104d5264/pybase64-1.5.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c1a3279af228faca3c224cc8c30aa130b5f3184ba420ac477de1db2cb99be8a7", size = 92538, upload-time = "2026-08-08T15:39:45.863Z" }, + { url = "https://files.pythonhosted.org/packages/ce/fe/5775be12186b4bdadd9b7bceea9871af097547d5a2d8bec4e43ed9d5408e/pybase64-1.5.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:d8e05ac71573089f25cdbad4b01db8d0b8e82846cd42291ef002d265903b1e41", size = 87146, upload-time = "2026-08-08T15:39:47.04Z" }, + { url = "https://files.pythonhosted.org/packages/53/0d/15b1ff749dafa2146a0a7aabff8596cc6bbb4277fb90f56a0beca9cdda92/pybase64-1.5.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:08907ffbf8381a017f6332ce02b818e672c73563ec19f38a022a34fd1c55b493", size = 89500, upload-time = "2026-08-08T15:39:48.198Z" }, + { url = "https://files.pythonhosted.org/packages/8e/dc/bcfc83c650a83f814235c56c810570fddb382a23ad3f79c6816e0c9b4351/pybase64-1.5.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a6a5f053f077aa8f0ffe5d4d03dd7d3fae4b85155942228a6dd20b467c4d7d80", size = 88169, upload-time = "2026-08-08T15:39:49.455Z" }, + { url = "https://files.pythonhosted.org/packages/ff/43/992c2aa344020575b0539c388104cb9ef45c80429f99acd0f177d32bcce2/pybase64-1.5.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:1e149af6b5a5af697725abc52aefef7e3ab036f21f5c229848b0f8bc8f26edee", size = 84770, upload-time = "2026-08-08T15:39:50.619Z" }, + { url = "https://files.pythonhosted.org/packages/3e/c5/da628bb02323057cfb8653b427fb3b6c363a395e954dff38c512fdfeea56/pybase64-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:678cf90273ee5fa7cedb35334c765ced4dad38608c0258445da009c1da9dd174", size = 90101, upload-time = "2026-08-08T15:39:51.783Z" }, + { url = "https://files.pythonhosted.org/packages/c9/0e/5824a07fd5f64e80ea042624f0e2b03cdaa3ce786e201c02746b2552d4ff/pybase64-1.5.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:7dc71ba89766bef4bd2d9be8a827ce784f1c85915b8bcad2deefd7d892d6816e", size = 82843, upload-time = "2026-08-08T15:39:52.892Z" }, + { url = "https://files.pythonhosted.org/packages/b8/21/fb0e4da0de2e5bbd3a9bee14c6919550a6ebdb7344776b7730960a8d37b5/pybase64-1.5.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:c6b6c15473fff013dfcb0b89cfcbc922442459b08e96d37cdcf1a8bec28e4ed4", size = 97635, upload-time = "2026-08-08T15:39:54.095Z" }, + { url = "https://files.pythonhosted.org/packages/06/ce/c382f3401435e04a2440cbc6beb7317278baaf4e2fee28846325446f669e/pybase64-1.5.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:831c25fd670727aea65525b9d6cff00718f26ca92433f9ed039fe67af9825388", size = 86714, upload-time = "2026-08-08T15:39:55.904Z" }, + { url = "https://files.pythonhosted.org/packages/a1/90/f0552285b2e17ae405e675debf3fa6b999622b1cc3f72584b9cb3904584c/pybase64-1.5.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f2509dc39574f1a0c60eb5f6c968e6f064b55bea88506df25d15ba6d391b1c48", size = 84316, upload-time = "2026-08-08T15:39:57.268Z" }, + { url = "https://files.pythonhosted.org/packages/8a/d8/66b3468adf62bee0c27cf39aa1f2da6d9b2e79f01d25c7af0310997502b4/pybase64-1.5.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:fe57aab650c771802cc7b0eb541a74b6a181cd1870f61c537294ab462fec34e8", size = 85344, upload-time = "2026-08-08T15:39:58.423Z" }, + { url = "https://files.pythonhosted.org/packages/a8/f4/8fc0795eeaaed8a7d29068e465aa60516891e95854e3ef23231d28bc0766/pybase64-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:816fccccaa736743c19f8fd687def788e0c0813f8168f88c4d169827b6726d65", size = 100106, upload-time = "2026-08-08T15:39:59.754Z" }, + { url = "https://files.pythonhosted.org/packages/83/66/5bd414d1dd9aaa1b3c108108f1a9c0d3de6192d8a8753a674c084429a654/pybase64-1.5.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:09ba0119df1766bb43ae9774df511b396b89bde68a797119366aca1292f83eac", size = 39873, upload-time = "2026-08-08T15:40:07.232Z" }, + { url = "https://files.pythonhosted.org/packages/df/2d/047976dfb83c30be4849bfee783fc45d0e2fdec9115939acf220fc95e9b6/pybase64-1.5.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:e3ed723ed56d273b0e3a45c2583c5566ccb39cc5fd4d335bdcbe235f84e1a211", size = 40356, upload-time = "2026-08-08T15:40:08.767Z" }, + { url = "https://files.pythonhosted.org/packages/6f/0a/3927d8d51cfcaf603f0065809a90f31cc5b4f98386eb58f5ccb0fc28bafc/pybase64-1.5.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:dd1ace6dacffce5cdbe68a3b2efdf22e3c890a906d887075e10dcc5f4124068b", size = 46956, upload-time = "2026-08-08T15:40:09.917Z" }, + { url = "https://files.pythonhosted.org/packages/42/fd/40e2339ac17c4b877c9588867144ca15eaab4644e3f59e494b831d73a770/pybase64-1.5.0-cp315-cp315-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:40399e568324635235697b00410634e0fb027432e9b9fef92886eb3407a5c211", size = 90912, upload-time = "2026-08-08T15:40:13.823Z" }, + { url = "https://files.pythonhosted.org/packages/82/e6/5eea2e16c31af2f5c31a7df903df05fb94e146b80709252b0e4da3a09cd9/pybase64-1.5.0-cp315-cp315-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:92dbad4599d5d081f905bba43b10690cc4d445857d04a7b18eba1a09bfa27cf6", size = 95172, upload-time = "2026-08-08T15:40:14.991Z" }, + { url = "https://files.pythonhosted.org/packages/72/45/502b486cd801297984f302558973364669b97d1a50e9867e12afa30b5d86/pybase64-1.5.0-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1e571d2db1c515641e9918cf04f23be58818ba6d56f266fab31dfc6d5f6e01d9", size = 84916, upload-time = "2026-08-08T15:40:16.276Z" }, + { url = "https://files.pythonhosted.org/packages/7a/00/2c9100bf1f651cf3a6a2c835306603323e3d05c9d2fe7d8ee3727bb7a718/pybase64-1.5.0-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:1e3f5f726bedde8d7006c4f8d61f0f053de65b806af24110278c530445b6da50", size = 88363, upload-time = "2026-08-08T15:40:17.636Z" }, + { url = "https://files.pythonhosted.org/packages/40/9e/fb4520a0cd238065141a89f9f2b6c54ef4e9ff6578ffba6122b5f9af24b1/pybase64-1.5.0-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:107129bf5591f040cd6cfe3b7ea5c1626a2f9610763e54d450778c578ca2b69a", size = 84714, upload-time = "2026-08-08T15:40:18.809Z" }, + { url = "https://files.pythonhosted.org/packages/c4/6b/87fd652e2c63516dfae373b9563329af2c5baf67a29a499601861cf52d89/pybase64-1.5.0-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:e161a4ba46caaa9417d5cd55f23c0717d5243b4f2a96c176b0d1a07bf86e0b0c", size = 83687, upload-time = "2026-08-08T15:40:20.192Z" }, + { url = "https://files.pythonhosted.org/packages/f1/ee/b083e2092dd1b6a6a973ced22b3363dc0bb27e7e2b21da8d83b44097d523/pybase64-1.5.0-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:741f944bef8dd709e9ca9e991f5f6a91a8d49b6e2725fdb4070027f0ec06faa2", size = 88276, upload-time = "2026-08-08T15:40:21.418Z" }, + { url = "https://files.pythonhosted.org/packages/11/ff/498c12d1316594d3796795e66301e193afa4e51fdd7e378c087922bfb074/pybase64-1.5.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:4c94d6b104411d33df813b1defa8a1194a884e9393839fefa3f7ea7377e1efeb", size = 83045, upload-time = "2026-08-08T15:40:22.974Z" }, + { url = "https://files.pythonhosted.org/packages/da/03/755d6316c7cfdab904311900aafbbbf1ed2227ae814bd3d2f25df8d10d46/pybase64-1.5.0-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:0976e9b7465387038868c6b560d7cdcbb9ef5214faf55ae6036e4aa4e93ba423", size = 77577, upload-time = "2026-08-08T15:40:24.242Z" }, + { url = "https://files.pythonhosted.org/packages/8f/c5/ebd26ad4032442d6fa6ba14ed0a222bd5d81f2a373d4d83a840a432e6c15/pybase64-1.5.0-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:6fa782fc5d7d53bb4c1b01e34909287f301c4c81251f8130e55848ab5d2f23e1", size = 90946, upload-time = "2026-08-08T15:40:25.462Z" }, + { url = "https://files.pythonhosted.org/packages/bd/5a/46f96588a494ab8ab71a3fae5a18627c728c0379ae96af96312e54f8c8e6/pybase64-1.5.0-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:d3e26e250aa51813881d03c09995a41462e115ab9c3c2b6d5202e4286b924d00", size = 82163, upload-time = "2026-08-08T15:40:26.745Z" }, + { url = "https://files.pythonhosted.org/packages/21/29/263bb998064c5d18957f5445a381f336070d24f09e7c372a0e3963dea142/pybase64-1.5.0-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:f4135c1e12615fa7989c9aec4720cedaa342bc4b8dbd5665f84a95790e3db5fd", size = 87170, upload-time = "2026-08-08T15:40:27.958Z" }, + { url = "https://files.pythonhosted.org/packages/e5/17/e5b3e991ebcc71d20ae9246011dde389e321f450b658976be7a51ca50824/pybase64-1.5.0-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:6ae263c1244bf375420fcdfd5ab32d463496f3814177edc8f0f3a8b56d7fe643", size = 81041, upload-time = "2026-08-08T15:40:29.641Z" }, + { url = "https://files.pythonhosted.org/packages/6b/37/a6e17849a37cb94b010b4eb7decaa5b49b6fafd0d18b386bd1cbe4b4d523/pybase64-1.5.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:d0930504fbe5c003f31d67aeab4b8f155a409168a26ef8ea7df759bc50ab6729", size = 93974, upload-time = "2026-08-08T15:40:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/27/58/9d9f38918c9b27ac7200302fde0baecb95daf3b8a3cbc917238291691134/pybase64-1.5.0-cp315-cp315-pyemscripten_2026_5_wasm32.whl", hash = "sha256:9edbf7e7a97454904a4ccfbd007a511b75ebf13cba9d0dbdfe6c4480e154edf6", size = 33157, upload-time = "2026-08-08T15:40:32.094Z" }, + { url = "https://files.pythonhosted.org/packages/3f/da/1cdd664628bef3b6108fac20e2a11df8992e1b0d5a1ff1336256d8817961/pybase64-1.5.0-cp315-cp315t-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:f8dcf39b6aabed5d3820188451e98d651a9fde2453a2e99fb386941d4bd518d9", size = 97813, upload-time = "2026-08-08T15:40:39.505Z" }, + { url = "https://files.pythonhosted.org/packages/fe/56/430bd2afbd179a278c87e282062db898735a9dc17255b223f1c0d4276b5f/pybase64-1.5.0-cp315-cp315t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3ee57900cb5d35a79d992800103180d715b68d8b56658b445a10f97e8805982", size = 102425, upload-time = "2026-08-08T15:40:40.745Z" }, + { url = "https://files.pythonhosted.org/packages/e8/a7/6a80063f72ba5bf09bc43a26ad5bb6152a1fc52fec75f7b24d40ec25c37c/pybase64-1.5.0-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5d1c9d46d6b8459f5dac87b1778950ad28e27a83d1cdba1d2c34a031dcd57e2", size = 93851, upload-time = "2026-08-08T15:40:41.957Z" }, + { url = "https://files.pythonhosted.org/packages/0c/b3/dfc900a5a724452defb33dff71a869638e4e58497dc7fe20602d6e650b64/pybase64-1.5.0-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.whl", hash = "sha256:84e619e315fdaf8b70d54cdd0be12c7895dcdcd0212a42a67576b33f7af111dd", size = 92634, upload-time = "2026-08-08T15:40:43.164Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/da8aeb775098fd53d55c289089e6fc94b37751d156e130d11f8c137caf8e/pybase64-1.5.0-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:80eb2c568f1f09283ad7528407a97e84935f23851943ed27206b52664b8010f0", size = 90805, upload-time = "2026-08-08T15:40:44.357Z" }, + { url = "https://files.pythonhosted.org/packages/df/93/28a50e85bab801e642762f71d852ae765970a0df8b9915848e822b73d64a/pybase64-1.5.0-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:69a2c6eaaa3b7e157ddd1c3803d09e5fa80d9aeb5191b81ad60e182662c2a324", size = 89140, upload-time = "2026-08-08T15:40:45.599Z" }, + { url = "https://files.pythonhosted.org/packages/a0/17/17c43a329d20a299b25a01a425dfd5f671274a5ad65754ca314720ca9f24/pybase64-1.5.0-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:e1df96c88f8e9f57cbe25f0d8f28411e2d1cc42be26e99078f6e4efa876dcb96", size = 92235, upload-time = "2026-08-08T15:40:46.838Z" }, + { url = "https://files.pythonhosted.org/packages/11/97/e42e428a4da55f56d873afc555ff18a91e2932a6a44b4367a9c072d09c03/pybase64-1.5.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:16201c0998c80f0bac0817a792969b7e1f4169014a8a6b32019e005384734805", size = 91740, upload-time = "2026-08-08T15:40:48.108Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f5/4a527a34c2742009376dba884e84b0e34e44253f5e6b951c66494dff488d/pybase64-1.5.0-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:f5d28afc34ee925f0beb376d2e3ace38267e700994481511686f2b467f11f51c", size = 85993, upload-time = "2026-08-08T15:40:49.352Z" }, + { url = "https://files.pythonhosted.org/packages/43/9b/50bda3bd73f0f20e83a7941b98e5c655ba1cb6d0d9228c192e3b2ee7ea56/pybase64-1.5.0-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:dc719c38087e09788d40216ebaacc89504dd8e964c0457085a4c1b83695eaa5b", size = 97768, upload-time = "2026-08-08T15:40:50.593Z" }, + { url = "https://files.pythonhosted.org/packages/11/87/befa9e85b22f32b8eadbdc1145f61ebb16d571923954ac258ddb7f96958f/pybase64-1.5.0-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:7b809817bf0413bcc00cab69d6a055e1fb2626b22359772c2c3570ac3fef7462", size = 87872, upload-time = "2026-08-08T15:40:51.91Z" }, + { url = "https://files.pythonhosted.org/packages/c6/b0/5b213e770f5e8f3df9a09d4337821a7ffd5001e56a248ebf782a6a8bbce7/pybase64-1.5.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:e953b14d562b7c08eae7b7c327b5162c78a6975974d8de8d7acff2b8b7c682b0", size = 92278, upload-time = "2026-08-08T15:40:53.153Z" }, + { url = "https://files.pythonhosted.org/packages/75/58/1751447e2f3d480dad36d9d0f4a18a65062551ada9cf5a18599a79583536/pybase64-1.5.0-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:8a5aaa4343b5ed1af3850ce351482e7385d695af15b81b244c3f823949dfe796", size = 86299, upload-time = "2026-08-08T15:40:54.516Z" }, + { url = "https://files.pythonhosted.org/packages/9d/2f/d505ad3dab5ae4339d0ae471453a305ea7cbe9be630825fb06019d18fe0d/pybase64-1.5.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:a80b502057361c8f2f5f9b75ecda9127b4ea1b1baec7b99b63d425c09e799b12", size = 101313, upload-time = "2026-08-08T15:40:55.813Z" }, + { url = "https://files.pythonhosted.org/packages/ba/dc/cd57bd8629965d69eaaa721cf915f3c0590ba468811d290bbcdd3908f0ee/pybase64-1.5.0-graalpy312-graalpy250_312_native-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b618ecec8f13b3f9dd58e257aa98fc9b017829a1bdc4f576e9146998956ec2c7", size = 54270, upload-time = "2026-08-08T15:41:29.872Z" }, + { url = "https://files.pythonhosted.org/packages/aa/22/67ad2ddf8ed03e0fc94341ebfc6ed694a36b9c908dd5a08b3ca366e31892/pybase64-1.5.0-graalpy312-graalpy250_312_native-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d09d63b219adfb1b40e104036dc2462234d2f06c05e436918e08f31a09a973b", size = 45919, upload-time = "2026-08-08T15:41:31.177Z" }, + { url = "https://files.pythonhosted.org/packages/86/d9/399c45ada7e401c927345324baf797b167864b9817be6aa71a1e28a00ad1/pybase64-1.5.0-pp311-pypy311_pp73-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:adfc52cee3ad56c070e824bee9feda1f13c8679601ff8d0535f03da60bdcdda6", size = 50312, upload-time = "2026-08-08T15:41:45.607Z" }, + { url = "https://files.pythonhosted.org/packages/62/18/064288c6211c79a27893b70261558cf79a254ca22d80101bd7d05a817a6f/pybase64-1.5.0-pp311-pypy311_pp73-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2d240554e1a63ad9b7cb128acf94d4bc7d8400c78dfb76521775e767d4aa0b22", size = 50782, upload-time = "2026-08-08T15:41:46.855Z" }, + { url = "https://files.pythonhosted.org/packages/a8/73/d68d5df0f367613643c9cb9470a25611d2d078471c1eadeb86c00e644182/pybase64-1.5.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b7af9ad847b351b42ec54b3c0580febe406b28408917b7fc1565c87896ed0c4d", size = 45251, upload-time = "2026-08-08T15:41:48.082Z" }, +] + +[[package]] +name = "pycountry" +version = "26.2.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/1d/061b9e7a48b85cfd69f33c33d2ef784a531c359399ad764243399673c8f5/pycountry-26.2.16.tar.gz", hash = "sha256:5b6027d453fcd6060112b951dd010f01f168b51b4bf8a1f1fc8c95c8d94a0801", size = 7711342, upload-time = "2026-02-17T03:42:52.367Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/42/7703bd45b62fecd44cd7d3495423097e2f7d28bc2e99e7c1af68892ab157/pycountry-26.2.16-py3-none-any.whl", hash = "sha256:115c4baf7cceaa30f59a4694d79483c9167dbce7a9de4d3d571c5f3ea77c305a", size = 8044600, upload-time = "2026-02-17T03:42:49.777Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -3242,6 +4927,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/87/b70ad306ebb6f9b585f114d0ac2137d792b48be34d732d60e597c2f8465a/pydantic-2.12.5-py3-none-any.whl", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d", size = 463580, upload-time = "2025-11-26T15:11:44.605Z" }, ] +[package.optional-dependencies] +email = [ + { name = "email-validator" }, +] + [[package]] name = "pydantic-core" version = "2.41.5" @@ -3339,6 +5029,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] +[[package]] +name = "pydantic-extra-types" +version = "2.11.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/71/dba38ee2651f84f7842206adbd2233d8bbdb59fb85e9fa14232486a8c471/pydantic_extra_types-2.11.1.tar.gz", hash = "sha256:46792d2307383859e923d8fcefa82108b1a141f8a9c0198982b3832ab5ef1049", size = 172002, upload-time = "2026-03-16T08:08:03.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/c1/3226e6d7f5a4f736f38ac11a6fbb262d701889802595cdb0f53a885ac2e0/pydantic_extra_types-2.11.1-py3-none-any.whl", hash = "sha256:1722ea2bddae5628ace25f2aa685b69978ef533123e5638cfbddb999e0100ec1", size = 79526, upload-time = "2026-03-16T08:08:02.533Z" }, +] + +[package.optional-dependencies] +pycountry = [ + { name = "pycountry" }, +] + [[package]] name = "pydantic-settings" version = "2.12.0" @@ -3640,6 +5348,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/1b/5dbe84eefc86f48473947e2f41711aded97eecef1231f4558f1f02713c12/pyzmq-27.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c9f7f6e13dff2e44a6afeaf2cf54cee5929ad64afaf4d40b50f93c58fc687355", size = 544862, upload-time = "2025-09-08T23:09:56.509Z" }, ] +[[package]] +name = "quack-kernels" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi" }, + { name = "einops" }, + { name = "nvidia-cutlass-dsl" }, + { name = "torch" }, + { name = "torch-c-dlpack-ext" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/75/5c/67c4a0d54cbb8d4d05af73132d620d1900a193b33e0d5ea6a25829b941d2/quack_kernels-0.6.4.tar.gz", hash = "sha256:8bf08a0e1a85aecc892ce84f4b6ed7abb6a44ec7c68b83040abe174bc8c2b936", size = 845485, upload-time = "2026-08-07T23:25:31.94Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/f5/36ec7e075e90bd92bdf30fee0606f189348c2bbb0af6aebaf7382927c622/quack_kernels-0.6.4-py3-none-any.whl", hash = "sha256:e77c5d1f1299b0b38487fe8737df6c6975daa16bca7f7eb883bd1a74d09e7e78", size = 728458, upload-time = "2026-08-07T23:25:30.549Z" }, +] + [[package]] name = "referencing" version = "0.37.0" @@ -3832,6 +5556,116 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/2f/b4530fbf948867702d0a3f27de4a6aab1d156f406d72852ab902c4d04de9/rich_rst-1.3.2-py3-none-any.whl", hash = "sha256:a99b4907cbe118cf9d18b0b44de272efa61f15117c61e39ebdc431baf5df722a", size = 12567, upload-time = "2025-10-14T16:49:42.953Z" }, ] +[[package]] +name = "rich-toolkit" +version = "0.20.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/3a/a258c2fbc6c6bdf428611388f5698ba5d57ffdf0755e1cab474d9cc47813/rich_toolkit-0.20.3.tar.gz", hash = "sha256:223dd2cfba325ed55e94933b9e53f3aca13e9fdf76622bd564c18109a2273c1b", size = 205355, upload-time = "2026-07-13T14:38:06.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/ce/639d0d0ce3d25c5edbd1afecd308bb35dc04883a45ac9f0855c8aee4e919/rich_toolkit-0.20.3-py3-none-any.whl", hash = "sha256:419aa87516d5f3849cca553c6dcf707c02a36d508fcf996946606725d34a3002", size = 36195, upload-time = "2026-07-13T14:38:05.687Z" }, +] + +[[package]] +name = "rignore" +version = "0.8.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/7e/aa0640d74f6b4bb68466f5899bd5ed1680480732344c31a408504e215801/rignore-0.8.1.tar.gz", hash = "sha256:2b6cf58501e9ff1b6a71c3fd66c8a105311e1f23237626fd4c9c00606bb3d30f", size = 55535, upload-time = "2026-08-04T22:27:08.237Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/35/ddb14543c0fde4a950d2924a19959ffbcfa6b3088c862f9d9a13e76acb54/rignore-0.8.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2aa76742832db1fdac070fa8e694dda0daf6f2f7191832bd555d1e80f8c9c42b", size = 887919, upload-time = "2026-08-04T22:23:12.353Z" }, + { url = "https://files.pythonhosted.org/packages/d2/27/2fdd67883dec9b9a58c433c5af1d7776064786f1b0078bb07799682ce310/rignore-0.8.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a6c53ebb7ea2d22116d99ec155c5313c3d76220f06b4118c1e1de629de721a95", size = 861417, upload-time = "2026-08-04T22:23:13.712Z" }, + { url = "https://files.pythonhosted.org/packages/40/45/bd9df565ae9d5a751f10c05e660d4a0640414385e440f80c5d2c6c85ca1c/rignore-0.8.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0dbea2e28538fa64f8409085abf50240f1f7c5981a5479bbb1aea7b84d44aa45", size = 1139170, upload-time = "2026-08-04T22:23:15.054Z" }, + { url = "https://files.pythonhosted.org/packages/86/d9/dfa38aca07aaa401c1cb07b5810d68dd84a4fc81e44a0f1da2daa97dcdd1/rignore-0.8.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:05fe665768bfe8ddda77e5ef1c2332555ea2abecd33aad05fa41a19d959ba011", size = 917578, upload-time = "2026-08-04T22:23:16.422Z" }, + { url = "https://files.pythonhosted.org/packages/08/dc/dd73f3d47e3da48ff2535d24ee2dfda3ecacba9ad5f96b96a15860796041/rignore-0.8.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9869b618dc28104003292f26cad26a2d54f6a3941684f16fe9ff435b5df1b0c0", size = 932629, upload-time = "2026-08-04T22:23:17.915Z" }, + { url = "https://files.pythonhosted.org/packages/33/ce/d5fa33660a3dc494d6cda4dc012b2bd738e80825340ca313b8e37de72fd5/rignore-0.8.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:d59e4ee62d89136eb3f0f04db023c17f9294cde905cae2a0fc8d89a4ab86c57b", size = 898518, upload-time = "2026-08-04T22:23:19.21Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ff/d4f6aedfb0a0bbe29ddbf1ed8046de2fadb02d8db70334fa8a45b9a8c396/rignore-0.8.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:df64fa4f0b451cf198bcb1596bdce1d976396e579188141276d4ee8288ee2486", size = 970620, upload-time = "2026-08-04T22:23:20.572Z" }, + { url = "https://files.pythonhosted.org/packages/52/0d/c1b147143cbac03da2c6252482f9b62e95bb2935daf2949aeaf8766bb7bb/rignore-0.8.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a1fd861fe4c1fea39541e7c9e8a409472b9cf7604bb472fa9dc09c97bf7134b0", size = 1064904, upload-time = "2026-08-04T22:23:22.116Z" }, + { url = "https://files.pythonhosted.org/packages/62/43/8e491db960cc9cb553ab75044faf7d0dc4f178ad5a9cea8c44c1b2dcbabf/rignore-0.8.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:c49ea7bac11f3bed7af4a21fb76dd7820341103dd3d68676a7177d797262b84c", size = 1136229, upload-time = "2026-08-04T22:23:23.763Z" }, + { url = "https://files.pythonhosted.org/packages/5b/b5/05e729987b85a4cd7422f06159554c50060c1338bebc9a19b26f8e34ac8c/rignore-0.8.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:514f7be6d04a6f6be4835748b14bdb027799759d02ada03ead064a50c763b70d", size = 1146385, upload-time = "2026-08-04T22:23:25.25Z" }, + { url = "https://files.pythonhosted.org/packages/9f/e6/d7aec128c2c26fe029e3d05e919d1dbe9fbcc108f46e1a9e71f5955ddda4/rignore-0.8.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b493c787dd2cc31b1e1c678e626bde593d83f9df420096f91dee6841ca317bfd", size = 1142766, upload-time = "2026-08-04T22:23:26.565Z" }, + { url = "https://files.pythonhosted.org/packages/14/df/201b3ec49a4714fcad4515a6e1f2d55d3c9e3d8b43a5eb9307a8dfbff506/rignore-0.8.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac0a7cadcf6154dd60b2f101425644179b34c540a88b84089995c1745e8c623e", size = 884566, upload-time = "2026-08-04T22:23:34.516Z" }, + { url = "https://files.pythonhosted.org/packages/3e/f0/c4a64587c650f8c28c3d8afec3ce6a628a8d0ea41ee8af58389e5882d3fd/rignore-0.8.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ab53a0908a1f24d2aaf4201920932134fbeb19be1d4c9619514ef7c781c9f3cf", size = 857414, upload-time = "2026-08-04T22:23:35.737Z" }, + { url = "https://files.pythonhosted.org/packages/00/6c/b8345ef35e5cd672df57a529f44c7f01a38daf3fcbd1afb7046290254c3a/rignore-0.8.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dd43540d294fbcb66daf66836b6c57043c0d3af672722099cd4a0c91448d948e", size = 1132587, upload-time = "2026-08-04T22:23:37.109Z" }, + { url = "https://files.pythonhosted.org/packages/ec/fe/ac83dc97554f183d70a02c202e089ea91aaddabddc173b0cdf9f23ce022d/rignore-0.8.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b4316b266e88c25ac7b4a0c765aa93c436dfa689a29429bd276878db45e17153", size = 913989, upload-time = "2026-08-04T22:23:38.356Z" }, + { url = "https://files.pythonhosted.org/packages/a9/a8/c6e6db62226df8cd3d950889925d826a0e35ac567cc5df548a872944f1ed/rignore-0.8.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2e68cfc4ee0a2909952af2aebd608d1cd22f7d1cdce332cb0b5ea3762939865d", size = 927835, upload-time = "2026-08-04T22:23:39.887Z" }, + { url = "https://files.pythonhosted.org/packages/9d/dc/850c839ba4ffef76bd4ead7c3556c5e38d16f4cd9d581d01cb97496d3690/rignore-0.8.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:4d891dbe52b1aa4df22a69346e731ee3df219128956ca4391722773e6baab16c", size = 890590, upload-time = "2026-08-04T22:23:41.358Z" }, + { url = "https://files.pythonhosted.org/packages/9d/a2/a17d83b8923dad2d809c515b6dae89304e4057fc9422c6cc7af0a6ce4c3f/rignore-0.8.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:463f9734f06f2dd939c45f0e58d50c0648f799e9c41fb657e40d236054b14cfc", size = 963504, upload-time = "2026-08-04T22:23:42.804Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d8/a5147a1c60f73dedc61c3af1a1de36f11a4b2322d7482d4ef4306e6915a0/rignore-0.8.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:97fceadfe03f3b8cd77cceeed92d00fad67f3ab80a0fd4d31e8dec104b721018", size = 1060732, upload-time = "2026-08-04T22:23:44.437Z" }, + { url = "https://files.pythonhosted.org/packages/67/33/7f5e99e2b63725ae843daedfabe02d14c513778ede4dd4ca91a1771d5107/rignore-0.8.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:43342bf37e7bb57d69f766678c2b19fecf2b3ec757be3f1bb2fb774d2be8c81a", size = 1132151, upload-time = "2026-08-04T22:23:45.967Z" }, + { url = "https://files.pythonhosted.org/packages/c0/0a/45348d4292464afaa5d659d3afa64580dd74cd06bb0de327159b1f9ed07f/rignore-0.8.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:514000eabd84e8f6f6a589a30d00e76d7cebe0d99b6cacede1223f64dca4b742", size = 1139846, upload-time = "2026-08-04T22:23:47.493Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ac/33e84dc397275787d11849b5be0175c297d6210a9d3058d1c1b56c945363/rignore-0.8.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8dfa13e24fc32d3df33d40788cd04e07f6b607f3c5306808ee0c285d2135effe", size = 1139121, upload-time = "2026-08-04T22:23:48.919Z" }, + { url = "https://files.pythonhosted.org/packages/83/94/d87afacf13f5e32a844ffdd2d906a3c8d5ce24de0a91ff02a605322f34bc/rignore-0.8.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c5ce3b10ce4b716abc535bc0fdd66b0b9fa9f5b1987b36d3f5ece7d0e2a9a81", size = 884581, upload-time = "2026-08-04T22:23:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/cf/ca/ea796f9ad84f8b671d682e27bd1a60a7679b844faaf582b11f7e527967f1/rignore-0.8.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c199fa2d4a898e9b686d846371ead3c8e08d3e29f78e2ecabf4c580820a8f764", size = 856803, upload-time = "2026-08-04T22:23:58.456Z" }, + { url = "https://files.pythonhosted.org/packages/75/71/0f3e6d0c421c7a5f998a21d3314f76f26793bc3af851a5e43b1e82d290aa/rignore-0.8.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7136b7ff29c37c8ec8effa3c59e27839482ba542af94fdc2581a59405e99037", size = 1133897, upload-time = "2026-08-04T22:23:59.923Z" }, + { url = "https://files.pythonhosted.org/packages/20/7a/0861528542be468f493e809064842fbb595706c0adce02d3a6e56f15ad2e/rignore-0.8.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32b0fcd01495cc4f4d10b5307f9f15818c9db752cfbd7a9ecb00b0f129a70dc2", size = 914178, upload-time = "2026-08-04T22:24:01.316Z" }, + { url = "https://files.pythonhosted.org/packages/1d/19/769c0a832d0f0a8d368f09ef7dee4caf53d46f391de489532bf44e1eee1d/rignore-0.8.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ca91c91a53135889945e77b286215ecd41c8f1090398db97ca3459c5a290eb6", size = 928035, upload-time = "2026-08-04T22:24:02.735Z" }, + { url = "https://files.pythonhosted.org/packages/50/e9/76cf08722a4d93836f7d416877f1a0d8e9a456f2e145ab57f12b301ccf70/rignore-0.8.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:67e30c0883f9aef3bcc45e5dc6980bc41f161da6c1b1a460f0bd87064c1d6594", size = 893393, upload-time = "2026-08-04T22:24:04.306Z" }, + { url = "https://files.pythonhosted.org/packages/d4/42/ede8c973b1f979b81a4cc48530107538aa9ec7df97c98759415f958bb0eb/rignore-0.8.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3012fc79b19953f76a67b2e6bae5f456742e96f0ed33d3c6c9390b0586fc2da9", size = 964677, upload-time = "2026-08-04T22:24:05.965Z" }, + { url = "https://files.pythonhosted.org/packages/57/22/70af384dde865285d71f5deccacedc89232a7e04fed558d035dd31d001af/rignore-0.8.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3a437c870f1465aba36eb4ac7108c09d5097c436123fdaec980e7a26a4595141", size = 1060291, upload-time = "2026-08-04T22:24:07.354Z" }, + { url = "https://files.pythonhosted.org/packages/9e/93/1c661799fb7270c55122efa0ac6211fb49b066579095765db90faed06af5/rignore-0.8.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:59f4f92ec5165619b3bc58130ab558bc618f9c5e9df06807339a85d965143fd5", size = 1131079, upload-time = "2026-08-04T22:24:08.708Z" }, + { url = "https://files.pythonhosted.org/packages/59/62/95655b6d2c6d2e82414f431b06fd7a3e6198adedfaf60de2b924c8f8d344/rignore-0.8.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:304ddf5f807c788c08a8210f9977131e7bfa9aa94704668d127689c00279ef41", size = 1140860, upload-time = "2026-08-04T22:24:10.149Z" }, + { url = "https://files.pythonhosted.org/packages/3e/bd/efd83ece6d828f908c751422849ef219847e084fb749f08aeb294e3a48d8/rignore-0.8.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1ce5c6d8f02badc55112b014d4fc9af662f0649869913d25785d7b3676ac19ed", size = 1139561, upload-time = "2026-08-04T22:24:11.633Z" }, + { url = "https://files.pythonhosted.org/packages/d4/08/cb82a725f040cc5d106a1fe33dec4031fb9d8863a4874a8dfb60c4db79f9/rignore-0.8.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6012766ea3a5a635d9b79f3e8c3797d5e47ce5e5dd81993c9f92a3ea4ff68b4c", size = 885260, upload-time = "2026-08-04T22:24:20.236Z" }, + { url = "https://files.pythonhosted.org/packages/06/77/6ba8d24fd151513347d9420fb383ec732b54886c6a5500e13f0bd7ec4072/rignore-0.8.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:90e60f0073caae0f1d59c993adedafa3a57bc6fac551669677cb25aa7fa9d9b8", size = 857548, upload-time = "2026-08-04T22:24:21.769Z" }, + { url = "https://files.pythonhosted.org/packages/47/66/c54f448b077e89de99e85746677425fbb3a62d3721aeb9afa556dc3ca10f/rignore-0.8.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:121ab7ac93e39fd1d70098461c1ed9a6fb89d54e7bf8c60ae351b23b05cbb8c1", size = 1135817, upload-time = "2026-08-04T22:24:23.179Z" }, + { url = "https://files.pythonhosted.org/packages/c3/92/4b47da84a36687f10c09b43de7600d7dde8f56f2981e68b8822cfadd26aa/rignore-0.8.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:befb772556c8463c640b290f632b57440182edd39996708a33c50bcf437796f9", size = 913513, upload-time = "2026-08-04T22:24:24.719Z" }, + { url = "https://files.pythonhosted.org/packages/2b/a4/8428734c0217b5c0adbe752afc9603ca34434c6033584f63d6ca7ba50331/rignore-0.8.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a184fa45db8cdc7a8604d2df020a107be3fc0adc2e522b4e4eb5cd5b57d5f84", size = 929324, upload-time = "2026-08-04T22:24:26.059Z" }, + { url = "https://files.pythonhosted.org/packages/da/f6/ab3738e42d9d351033b8e8fb46ac4b8eb193545aaf3ed2f52f58fb8d5212/rignore-0.8.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e68a572efc126aa45195f1581a5ea97c4e36eeba6874a6635ae44daa4fbec7a4", size = 891130, upload-time = "2026-08-04T22:24:27.374Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ba/9f374cf332eeea0e26cca6b6550678d0c69208e7ba4aea8ece0dc6b205eb/rignore-0.8.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c75ac1952ca8892422de328a925f4804120b149543042313bb3af7cbfdd65d64", size = 964769, upload-time = "2026-08-04T22:24:28.688Z" }, + { url = "https://files.pythonhosted.org/packages/37/46/7ee48265e34cb90ef4bcf85c63874da6d6a31af1c63fb7bca84fd5dbc188/rignore-0.8.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:3538084cef4a66ba3fee7c453d17db7cfb32a6653456381b62afd8d53090d6fa", size = 1061779, upload-time = "2026-08-04T22:24:30Z" }, + { url = "https://files.pythonhosted.org/packages/01/23/51c725ba23f06b2809a0a676127ca3613e852bcab0532e58cab9f09b6fbb/rignore-0.8.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:28a3baab3d1b7ea42c38e492d4a100de3e5c67217432364c62b5719e0f04e96a", size = 1132595, upload-time = "2026-08-04T22:24:31.472Z" }, + { url = "https://files.pythonhosted.org/packages/0e/a0/671192e2cc8711694cf564f77e9e59d5e8c41f100bce67fbb384f2befec5/rignore-0.8.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:74996d8ed1a494ff8e61d9194a0dd3637e6b19a582cbed6fe0e1a4cc60e7b266", size = 1141117, upload-time = "2026-08-04T22:24:33.148Z" }, + { url = "https://files.pythonhosted.org/packages/01/df/b3a9a821b37361debc29fd009d21dc4af7f8313b76371c1640aff2765a59/rignore-0.8.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a2e5df9ba53e502f676b054c4d12371273204e2b9c6ec29afad3a155b9ad3399", size = 1140515, upload-time = "2026-08-04T22:24:34.708Z" }, + { url = "https://files.pythonhosted.org/packages/a9/3d/1a260407a17e062859995d9f557b4ccbb36e6a0ef44a0d99409c836e5942/rignore-0.8.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f020b018577d0081a2b23da199d22c58f4cbd63935c8e991adffbb1a755b467", size = 884941, upload-time = "2026-08-04T22:24:43.414Z" }, + { url = "https://files.pythonhosted.org/packages/b3/80/70291634f2f2257fcabe1156926d7a10a87b0da1a8ba9ab2cafac37a4ded/rignore-0.8.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4d4a70c9b857657c3a542fd367e16ae106fb6e3e4448cddb30e4edc604d5a025", size = 856667, upload-time = "2026-08-04T22:24:44.769Z" }, + { url = "https://files.pythonhosted.org/packages/1d/84/3811795d0b6fb3e8865d6a4f4eb3d3184e68a8557d7ee5b6d5a463196f54/rignore-0.8.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ab668664e6388afc08fec50186ec21daf17c738ee71bb039d12cf5f14e964dde", size = 1134272, upload-time = "2026-08-04T22:24:46.213Z" }, + { url = "https://files.pythonhosted.org/packages/59/51/6a60ea8291346459e3f788bac4ecc9711b13271804c6eef47dc2ec59e6e7/rignore-0.8.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a3454cdd8bc145fd055dd45f650ccf1509e7b4edfc720152eb2be594b230f03", size = 914263, upload-time = "2026-08-04T22:24:47.604Z" }, + { url = "https://files.pythonhosted.org/packages/50/72/620d4baa44e44ee53ae516efcb714183fd592ba923dc7990bf57947ff610/rignore-0.8.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:425b962f3d68b86ebb785409153708a38b0658542c918ee706ea433e2547c805", size = 928135, upload-time = "2026-08-04T22:24:48.987Z" }, + { url = "https://files.pythonhosted.org/packages/50/60/06495efc96c2edcda647dbf7db770ab9307459d3b2f8598d9ae1f1ce9ea5/rignore-0.8.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:5094965794d163e1c4a71dccf3d1c3b3df86802e3afbfa461b300aa923cd7ea5", size = 890677, upload-time = "2026-08-04T22:24:50.318Z" }, + { url = "https://files.pythonhosted.org/packages/ee/92/396f751e69d543bfb7e9faa5b0ae3ee988fcd76d9c743857fdb26db0651b/rignore-0.8.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1467756e8454d3f816131bbad8b0efc52b7f6924e9b151f2962cec9e2f4af706", size = 964291, upload-time = "2026-08-04T22:24:51.836Z" }, + { url = "https://files.pythonhosted.org/packages/4e/0b/642483058dfbcbfc319d9cf48a05d8dbf9f7f6939b410eebdbc73dd9414c/rignore-0.8.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:b5acf12993258f0eac4db81c30bed5469ebadd1b5bb2863c985fe0d802d94e0f", size = 1060972, upload-time = "2026-08-04T22:24:53.921Z" }, + { url = "https://files.pythonhosted.org/packages/64/2d/69b54a53d94baf2fb16ea1246f96bf093aae6a0e5d5c5cf5099c1fee4fb8/rignore-0.8.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:329ae0bea9598541cb818bbecb1e1f56fc1faae77828ebb3846ef2b24a050d05", size = 1130938, upload-time = "2026-08-04T22:24:55.395Z" }, + { url = "https://files.pythonhosted.org/packages/bf/36/c213d838c3c2237da8c51163a6c55acaeedc59759d4884d81ab6d7250721/rignore-0.8.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:4069fcdff01999cd2d5a426eccda45a8b31a3e0eeee8b5a9da5a452714cbb2c7", size = 1140646, upload-time = "2026-08-04T22:24:56.741Z" }, + { url = "https://files.pythonhosted.org/packages/96/70/28125520727d72e2878c1a6a3dbd333e57f27730b463d629d170e2da3b62/rignore-0.8.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cf28097a83e1237f87e11e33fb12def022e103cab2a20340a19753e8e222aebf", size = 1139989, upload-time = "2026-08-04T22:24:58.127Z" }, + { url = "https://files.pythonhosted.org/packages/38/5a/78311079595c96e01a2404eff725468e25fad16d2836408046ffbdb33a69/rignore-0.8.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e250726b08957aabcf4e28aef4ea18bcccbd95f240cebb75a74b424f067dce4b", size = 885906, upload-time = "2026-08-04T22:25:06.956Z" }, + { url = "https://files.pythonhosted.org/packages/fe/57/7d456d2dcc881e6bf08275222ae36fff58343ee0fc47e1b26722f97b38ea/rignore-0.8.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:62470f4d70d83f124381975614ed7c7db6f5c5fb4d777ee88f7bdb9a7d14b65c", size = 858240, upload-time = "2026-08-04T22:25:08.371Z" }, + { url = "https://files.pythonhosted.org/packages/64/3c/0f9bd7b9b3af37b216096e1dc81f9368d64c13625ddd34b4d595b316b47c/rignore-0.8.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d886d6bbdd0a1a3ef73bd38d6768cabc9b30bfd1be7b157b27a4ac7ec6c5244b", size = 1135145, upload-time = "2026-08-04T22:25:09.81Z" }, + { url = "https://files.pythonhosted.org/packages/83/06/f0087a520e3bf3d9532aa823ea141e5a076432b9d036bb506a99ddb534c2/rignore-0.8.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7ed31657a20df59b0bd63283d278e8678749aa0154ba98fae0838a9845599695", size = 914331, upload-time = "2026-08-04T22:25:11.369Z" }, + { url = "https://files.pythonhosted.org/packages/f0/65/8ac37040f162cb965520286ef48472a87f47cb137df4068a0d32634166dc/rignore-0.8.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e425a42f6601bfed266f76f3294bca48efcfdb10c3c0c279fb2f977b1e2cc2bc", size = 929202, upload-time = "2026-08-04T22:25:13.057Z" }, + { url = "https://files.pythonhosted.org/packages/e0/93/19b752fda8a56424ee156ef6a9799c582eb6d2ffb233c7b91e103b79d54e/rignore-0.8.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:ff09db59f672d929bca88ee7089d3697256967df76a2bab8b187208f2b517bc0", size = 891752, upload-time = "2026-08-04T22:25:14.773Z" }, + { url = "https://files.pythonhosted.org/packages/a9/94/a5db364ddb136360c7c4ba75cb56ba2f7cd8321b296602f6ea7438fc1a83/rignore-0.8.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8dbc5898945c027dc0ae451de0c983e20bd8fa8297f40ee38d0ce1a3ed924d4c", size = 964714, upload-time = "2026-08-04T22:25:16.416Z" }, + { url = "https://files.pythonhosted.org/packages/f6/11/63316cd5a87402a42c75b6ada0331745975f0274cabb6a8aa85c043468ab/rignore-0.8.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1160abcd855964a9dd69f3d603ef57be14b1ea51ebaf50b07f737a3f3a8b89a6", size = 1062141, upload-time = "2026-08-04T22:25:18.094Z" }, + { url = "https://files.pythonhosted.org/packages/91/6e/19b4bdde3f53bb0c9427d65544f92f864a892d695559423ce01fd14c1a31/rignore-0.8.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:be994859b2cbbc69338351bc9908dd7d049de232e6eab5e998ec1feac3faf785", size = 1133487, upload-time = "2026-08-04T22:25:19.855Z" }, + { url = "https://files.pythonhosted.org/packages/1d/d3/fc41ea3164001fe84a2ec8b84d7bb48e7a73ebdcac6d68961c80f874dd4e/rignore-0.8.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:f52098f3245c7557229d8253ea3a53a03de03704685aba8a1ccd24a5f004db70", size = 1141434, upload-time = "2026-08-04T22:25:21.316Z" }, + { url = "https://files.pythonhosted.org/packages/34/35/b9d415a7e90c37b28b2d5fd6a074a4a3d572d5a0e0b596df0b4b85af8a78/rignore-0.8.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:c676080bb70cbd5429052bb5a3827b35543876a42c792c6d713d9e162bdaa00d", size = 1140492, upload-time = "2026-08-04T22:25:22.763Z" }, + { url = "https://files.pythonhosted.org/packages/b3/f1/10a11bf9ebf03f7a8d9876ff023634cd4ce61845be9929f816ece4dfff11/rignore-0.8.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8db2be0d49175d5db4cd503f8dc62cbe687b9ebc71af4cede1c8b40bab8f4f4a", size = 884975, upload-time = "2026-08-04T22:25:31.906Z" }, + { url = "https://files.pythonhosted.org/packages/1b/97/c8dddb2dab738bd1b3a0b1639bc42d696bdd2765515a302620a35795c475/rignore-0.8.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:601ff49d8458a21d745908e35f792c02b45ecae4a37fa3cd1fc03fa065962bc0", size = 856951, upload-time = "2026-08-04T22:25:33.354Z" }, + { url = "https://files.pythonhosted.org/packages/a5/1d/ddcc1cddfb0a79c82eedbcd25ba0b014d346251cc08c1938b90951660cf8/rignore-0.8.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bb58e93997d546fad4cfcd730a54d7c12cd7a61fe7a9b31bfc75402403dc559f", size = 1134401, upload-time = "2026-08-04T22:25:34.845Z" }, + { url = "https://files.pythonhosted.org/packages/13/0a/519d2e081e0b89c705bb4b282ed7540c185accc748c97c40dd80a333eb3a/rignore-0.8.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0378cf77b8d64560e0cb433deada18438b36ef7933dd284dd65347d03c56c429", size = 914389, upload-time = "2026-08-04T22:25:36.658Z" }, + { url = "https://files.pythonhosted.org/packages/ed/aa/c1e857c4b8f9dd388c38c8ca430917f24f1de44d9279d4f5d9edbcc94a0c/rignore-0.8.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0097a8c35106997d2b52851f0888777efe10e34772140a9fdb018b2f99238159", size = 928845, upload-time = "2026-08-04T22:25:38.421Z" }, + { url = "https://files.pythonhosted.org/packages/5f/9c/a9651d49fbf2a336074df0a09d01ee2d968dae902e88e51e32d822f5e9fc/rignore-0.8.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:2a809a250f1532b93dcc52e173ee71adcafdb536a125532687666047c4537ac0", size = 891090, upload-time = "2026-08-04T22:25:39.966Z" }, + { url = "https://files.pythonhosted.org/packages/53/98/d3242c45edfdc059fb80b73be9e27dac34639da40002ebbcd5c9b9acb821/rignore-0.8.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:580e019787032b430b857335a66db53d7ae0a200586b3a6b5d3ef227648300d6", size = 965283, upload-time = "2026-08-04T22:25:41.796Z" }, + { url = "https://files.pythonhosted.org/packages/56/37/e19e70395c2e5a06ba67e0a51e86ede1b4269b8d3ba091963c4f0a950f63/rignore-0.8.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f922dcf01e6a7ad26adfcb7f42635ae63066b883d409876129db62297061327e", size = 1061223, upload-time = "2026-08-04T22:25:43.782Z" }, + { url = "https://files.pythonhosted.org/packages/bc/bd/24b737f97c542e4883cfcc557b08565adc7c698e30836b8bc9855676d749/rignore-0.8.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:4900bd1ba8938e5a5e601306504c7b8169783a2d44a8396c8fda4a9b659eed51", size = 1131772, upload-time = "2026-08-04T22:25:45.391Z" }, + { url = "https://files.pythonhosted.org/packages/aa/8d/bedc51ca98696998797af67163c039e0adc7835035fb929f10d33fb3d997/rignore-0.8.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:3d581bf107490abae4d40a9aa823a79e18c88a21658514ccaf95e42e85d278f7", size = 1141631, upload-time = "2026-08-04T22:25:46.875Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9c/946e55a7e5bc7b165faed3475451683580ddf8b45d63e67c39a0cdee5ce2/rignore-0.8.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:78490b93aec14a87fa4c23e634b1d1dcb2bf5dcb90923f3b03a3428892a13d9f", size = 1140328, upload-time = "2026-08-04T22:25:48.447Z" }, + { url = "https://files.pythonhosted.org/packages/b8/7a/ba90f5ed9849b5787f7d8341ee7ee5b9f960947cd651da713c764f7b5c54/rignore-0.8.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a82567032d81f559ba4a9f3e96ffcb48ae149dff2f44c58c966b5890eb248895", size = 889243, upload-time = "2026-08-04T22:26:48.719Z" }, + { url = "https://files.pythonhosted.org/packages/ca/71/23de6c34932b145e66de124efa6384c7ebe31e38402e1aabbab6588b94c6/rignore-0.8.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d9a9812dc25dcfb9bf2e546247f6e38b0c29a0d39864eb62de2a48b2ba41b961", size = 862703, upload-time = "2026-08-04T22:26:50.396Z" }, + { url = "https://files.pythonhosted.org/packages/76/e1/78a8e0380fefdaada4663d17098a989e43312c1efa83f19ab21bb0338213/rignore-0.8.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c735c5586f4306d9a62ad374093d6127b24ddadeaac322684e3e4c923eeb6a58", size = 1140600, upload-time = "2026-08-04T22:26:51.884Z" }, + { url = "https://files.pythonhosted.org/packages/fc/51/2bacec13cde1d9ec7113f133ed2ecbf45f9e8fbd53a7434e1fee5d16d664/rignore-0.8.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a249e142d2d313872e526949b9cdc5363041d64531f53553e702c00747225dc2", size = 920069, upload-time = "2026-08-04T22:26:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/61/10/befc052ff61a070dfb86330bc0e9b3860d2d1386c30b76ec527dc50390b2/rignore-0.8.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe7a7dcd84affbf312336991202c11922c13aff323e2857c5baf10afc0dd7f2d", size = 934482, upload-time = "2026-08-04T22:26:55.389Z" }, + { url = "https://files.pythonhosted.org/packages/a5/45/22e55ed298df00e15e851489750ed509ca9d2677f7c2cb9474edb8392ce2/rignore-0.8.1-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:08acabc3203c68a8bed7689c50b546a4ad7c9144ced2199c1261a149f7a3cef2", size = 900919, upload-time = "2026-08-04T22:26:57.564Z" }, + { url = "https://files.pythonhosted.org/packages/23/bc/fcfa184912efb34b2d1cb1938d6f96550e20a06b89ea75e3b51238a48cde/rignore-0.8.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4862e015736af87a4afa1a75023d28363ee34449928f8f5b521d1f9eabb0c826", size = 971779, upload-time = "2026-08-04T22:26:59.574Z" }, + { url = "https://files.pythonhosted.org/packages/52/b2/d1a2071f9c908309c01ee53958ff1e4b81852671a32f206dc908bbf5519e/rignore-0.8.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:c96c3a434b50a7663e011cf6060b0f9ba44fd956b310b230bbca9dd665bb1f53", size = 1066847, upload-time = "2026-08-04T22:27:01.224Z" }, + { url = "https://files.pythonhosted.org/packages/3d/73/463687fb803db4c97fbe3d54354a8030b908e9685f29f4556335dc65c94c/rignore-0.8.1-pp311-pypy311_pp73-musllinux_1_2_armv7l.whl", hash = "sha256:4cc3003a462778a75aadabde87edebc0e1d205c119c703c155bb03a4f3e8be23", size = 1137382, upload-time = "2026-08-04T22:27:02.931Z" }, + { url = "https://files.pythonhosted.org/packages/70/6f/6d90ff36a046d2d05c55c9f8d6287e3d5a8ebe0fddd50dce8872019d4462/rignore-0.8.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:0dc7765f94d6660574d648c89e28c9932248499d47dad49c2786f2e49231e1d9", size = 1147729, upload-time = "2026-08-04T22:27:04.729Z" }, + { url = "https://files.pythonhosted.org/packages/d8/de/d2c11ed977298cee41c23b6625da9f816ba9f7b91005dfb6674cc0bec09c/rignore-0.8.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:f7b929db3f8f3fa240b0f328453de0b1b747e51b7992a9198c8412d16f4f0027", size = 1144578, upload-time = "2026-08-04T22:27:06.473Z" }, +] + [[package]] name = "rpds-py" version = "0.30.0" @@ -3965,12 +5799,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f6/b0/2d823f6e77ebe560f4e397d078487e8d52c1516b331e3521bc75db4272ca/ruff-0.15.0-py3-none-win_arm64.whl", hash = "sha256:c480d632cc0ca3f0727acac8b7d053542d9e114a462a145d0b00e7cd658c515a", size = 10865753, upload-time = "2026-02-03T17:53:03.014Z" }, ] +[[package]] +name = "safetensors" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/06/f955dbbb1859e3bd23c8ac6141af5106e7ad5fedec4a3a6e3d60f94b7001/safetensors-0.8.0.tar.gz", hash = "sha256:fabaf3e0f18a6618d9b36560682562157f77c2b71fcffc7b432be2baed9d753d", size = 325846, upload-time = "2026-06-09T07:52:25.563Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/09/7d/65a7de0af421317bb36a067241e4235fff194eed60b961ed6d3f59a3fc60/safetensors-0.8.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a46e5ff292c356d6991e60942ba7f79817682d3a2cef0702136448cb9c4d235", size = 502844, upload-time = "2026-06-09T07:52:07.624Z" }, + { url = "https://files.pythonhosted.org/packages/91/4f/3175c9d75634e0e0dda0082794193521035edd7c70a6f212bf33ca06ddf4/safetensors-0.8.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4124502b78f03534117c848f87a39b8f31e577b15eff423bf8bfb95f2a8c30d0", size = 511823, upload-time = "2026-06-09T07:52:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/20/87/846c289e7aa2299eff406335717cf43ce8777194ece8aad75772e0411615/safetensors-0.8.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7bc0a787ba8a35be368ee3574edfa2b1ad389eebd0a72e482ae275490e3f6c98", size = 633461, upload-time = "2026-06-09T07:52:11.128Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/8d64d9df2c45d5ded401df889d0ad90882804ca172d79ec4f0df8f727fe0/safetensors-0.8.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:040070828e36dc8e122178bbbd5830ff9e97920affb84cbe0f46442497bed358", size = 545148, upload-time = "2026-06-09T07:52:13.603Z" }, + { url = "https://files.pythonhosted.org/packages/28/50/f203ff3a3ddfe19308efc83c5a3a29ed02bf786732ec35e68bf9162f3365/safetensors-0.8.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd6f3f93c9a0a7cc2788ee63fb763353d4bd2e89b0751bc78fcf7dda00bea774", size = 516040, upload-time = "2026-06-09T07:52:16.29Z" }, + { url = "https://files.pythonhosted.org/packages/46/fb/cdaed17ceb2948784fd9c36b6fd3e951b608547cea81a48e8ee6f8cfdfcb/safetensors-0.8.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:fcdd41ec4628fee5799f807c73c353629130fbd942aa23d83c623dd6c9d52d78", size = 513832, upload-time = "2026-06-09T07:52:12.37Z" }, + { url = "https://files.pythonhosted.org/packages/0d/49/1e15de264dcc3b77943d2d0c56a95809956883b1c2d6d585c792523f180b/safetensors-0.8.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8e9f537aa183a38ace122d27303dcd986b26bd2a7591f9181d7f0c396f4677ca", size = 559930, upload-time = "2026-06-09T07:52:14.743Z" }, + { url = "https://files.pythonhosted.org/packages/2a/43/bf38443278eab4b1be1fce2931e2b012ad9cb7df52ada751d0aab8f7659a/safetensors-0.8.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:87eec7ffed2b809f05a398a8becb7d013f19f7837cd15d9748580d6cf30dbaf4", size = 678670, upload-time = "2026-06-09T07:52:20.032Z" }, + { url = "https://files.pythonhosted.org/packages/72/e3/68cd3fa5b48488e84add63e04cb12f3bc28ae4638c06d4508c6e88823d0e/safetensors-0.8.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:4a95ae2b05d7726d751da4ebf626a2ca782b706e101bd894c95bc2450b1cffcc", size = 786679, upload-time = "2026-06-09T07:52:21.322Z" }, + { url = "https://files.pythonhosted.org/packages/29/4b/1c19c509d56e01f4fbb3d0a2e597450f6cc04d1d56cf52defb0a62dfd715/safetensors-0.8.0-cp310-abi3-musllinux_1_2_i686.whl", hash = "sha256:3ae091f16662658bdc019a4ff6cb4c085bb7d725eb5978b183ffd265863b6d2d", size = 765683, upload-time = "2026-06-09T07:52:22.594Z" }, + { url = "https://files.pythonhosted.org/packages/27/43/41c1621732edd934d868a00d1b891584c892a7b62a9aab82ea5a0a5623ee/safetensors-0.8.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8e080062fcde23be189565e1c3305d16751a218ecf9412c8601e64204eb6f846", size = 722361, upload-time = "2026-06-09T07:52:23.924Z" }, +] + [[package]] name = "scipy" version = "1.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/56/3e/9cca699f3486ce6bc12ff46dc2031f1ec8eb9ccc9a320fdaf925f1417426/scipy-1.17.0.tar.gz", hash = "sha256:2591060c8e648d8b96439e111ac41fd8342fdeff1876be2e19dea3fe8930454e", size = 30396830, upload-time = "2026-01-10T21:34:23.009Z" } wheels = [ @@ -4045,6 +5899,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1c/78/504fdd027da3b84ff1aecd9f6957e65f35134534ccc6da8628eb71e76d3f/send2trash-2.1.0-py3-none-any.whl", hash = "sha256:0da2f112e6d6bb22de6aa6daa7e144831a4febf2a87261451c4ad849fe9a873c", size = 17610, upload-time = "2026-01-14T06:27:35.218Z" }, ] +[[package]] +name = "sentencepiece" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/33/ea3cb3839607eb175da835244a798f797f478c5ddf0e8ecdf57ea85a4c70/sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6", size = 8218435, upload-time = "2026-07-12T08:39:34.701Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/3a/7839048997c7bc0c34c57526f539f835e20c7a57dc2a99f99579b11cdbef/sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e", size = 1324282, upload-time = "2026-07-12T08:38:20.342Z" }, + { url = "https://files.pythonhosted.org/packages/06/5f/9117bf854aef817ad0d0ee9310eed0308a7e529e7eaf2e80ad9cd281ef82/sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107", size = 1394242, upload-time = "2026-07-12T08:38:22.976Z" }, + { url = "https://files.pythonhosted.org/packages/19/90/cd798935668cff71d309d8ff10385844ecf216b1fe454f1993ed8bf2cb91/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a", size = 1325200, upload-time = "2026-07-12T08:38:33.689Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2d/37e3da037318a70066ded0d51bc2a7f35491ae6338dd993d5eb1503fc3b5/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da", size = 1397736, upload-time = "2026-07-12T08:38:35.771Z" }, + { url = "https://files.pythonhosted.org/packages/32/4f/31c1073314ad94466bca37d29581761d70110237ee3d46b0efece59a8c1e/sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8eed98514bffe5ecac37f493f91869c351fbb05629328bfdbc08502c6c094dc0", size = 1324980, upload-time = "2026-07-12T08:38:46.304Z" }, + { url = "https://files.pythonhosted.org/packages/59/b4/a0356fa04d6a14337a6e0e443556785a0422c53ec58baae6b9568120eb0f/sentencepiece-0.2.2-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64b656f025355cf8c51abe9fbe3848540756c6d7ca5e6791b1afa664bc24c7cb", size = 1397593, upload-time = "2026-07-12T08:38:48.302Z" }, + { url = "https://files.pythonhosted.org/packages/0f/af/c30ee2a9f99d51db9844acaa8fa0b611a97c2fa7116646fa43db3300b187/sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec27c152a1f1b24bc9168b55a5880f3c16e2334e697da6f55a1046a22405a3d", size = 1328625, upload-time = "2026-07-12T08:39:00.849Z" }, + { url = "https://files.pythonhosted.org/packages/3e/1a/4c6b39d03f5ba8439509adbd5a23c9538088a3cb679e7a47b911e8442bc6/sentencepiece-0.2.2-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:59d6588712101ccfcae9b03692be3aaae1514c2078666d7b05f15ba3a702e41b", size = 1398595, upload-time = "2026-07-12T08:39:02.86Z" }, + { url = "https://files.pythonhosted.org/packages/10/ca/1b6c251321901cbf8a2d2e48b8b70eb82a449011b766af52a228d0a90b6b/sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1402d8ee36f0d851cea8eee4dbb85fea14643b7503cf4d00d102eec0fe3ca719", size = 1325463, upload-time = "2026-07-12T08:39:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/24/b3/718847349da7b25c8220ed86d85b89080af94740b2d87a59198104ae5c51/sentencepiece-0.2.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8d44b20234905ff022b7d535f79d1f823ad7670c9851cc4f03cdc34787cdb3ab", size = 1398138, upload-time = "2026-07-12T08:39:15.564Z" }, + { url = "https://files.pythonhosted.org/packages/78/4a/2288f60e7283583ec0a0f16e72f9c8e68557d7e7a4b585d2cda4f9f47e64/sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df88b0c34f2fa909d322f7b06b1398e1e81af4b2f42a7b8e3556f928b25d1811", size = 1328155, upload-time = "2026-07-12T08:39:26.422Z" }, + { url = "https://files.pythonhosted.org/packages/26/31/5dd6882ebe899f741a5cfe40ff56c6efc06bc26ee287abdb723b671f409c/sentencepiece-0.2.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3f5851441ab1ef8634963a5100b733a8bbeefe623e0c5c005b1f1f3880e574cf", size = 1398307, upload-time = "2026-07-12T08:39:28.637Z" }, +] + [[package]] name = "sentry-sdk" version = "2.63.0" @@ -4058,13 +5932,58 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/57/cb205f7d93373120f666b9c5736dc0815524d96a9b278e7a728f018dc22a/sentry_sdk-2.63.0-py3-none-any.whl", hash = "sha256:3a9b5ddd403f79eb73bd670f75f04485819db53d28f76ced7bc09041cb0dfd6a", size = 495950, upload-time = "2026-06-16T12:45:55.819Z" }, ] +[[package]] +name = "setproctitle" +version = "1.3.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8d/48/49393a96a2eef1ab418b17475fb92b8fcfad83d099e678751b05472e69de/setproctitle-1.3.7.tar.gz", hash = "sha256:bc2bc917691c1537d5b9bca1468437176809c7e11e5694ca79a9ca12345dcb9e", size = 27002, upload-time = "2025-09-05T12:51:25.278Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/2e/bd03ff02432a181c1787f6fc2a678f53b7dacdd5ded69c318fe1619556e8/setproctitle-1.3.7-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1607b963e7b53e24ec8a2cb4e0ab3ae591d7c6bf0a160feef0551da63452b37f", size = 32191, upload-time = "2025-09-05T12:49:24.567Z" }, + { url = "https://files.pythonhosted.org/packages/28/78/1e62fc0937a8549f2220445ed2175daacee9b6764c7963b16148119b016d/setproctitle-1.3.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a20fb1a3974e2dab857870cf874b325b8705605cb7e7e8bcbb915bca896f52a9", size = 33203, upload-time = "2025-09-05T12:49:25.871Z" }, + { url = "https://files.pythonhosted.org/packages/a0/3c/65edc65db3fa3df400cf13b05e9d41a3c77517b4839ce873aa6b4043184f/setproctitle-1.3.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f8d961bba676e07d77665204f36cffaa260f526e7b32d07ab3df6a2c1dfb44ba", size = 34963, upload-time = "2025-09-05T12:49:27.044Z" }, + { url = "https://files.pythonhosted.org/packages/a1/32/89157e3de997973e306e44152522385f428e16f92f3cf113461489e1e2ee/setproctitle-1.3.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:db0fd964fbd3a9f8999b502f65bd2e20883fdb5b1fae3a424e66db9a793ed307", size = 32398, upload-time = "2025-09-05T12:49:28.909Z" }, + { url = "https://files.pythonhosted.org/packages/4a/18/77a765a339ddf046844cb4513353d8e9dcd8183da9cdba6e078713e6b0b2/setproctitle-1.3.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:db116850fcf7cca19492030f8d3b4b6e231278e8fe097a043957d22ce1bdf3ee", size = 33657, upload-time = "2025-09-05T12:49:30.323Z" }, + { url = "https://files.pythonhosted.org/packages/6b/63/f0b6205c64d74d2a24a58644a38ec77bdbaa6afc13747e75973bf8904932/setproctitle-1.3.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:316664d8b24a5c91ee244460bdaf7a74a707adaa9e14fbe0dc0a53168bb9aba1", size = 31836, upload-time = "2025-09-05T12:49:32.309Z" }, + { url = "https://files.pythonhosted.org/packages/d0/99/71630546b9395b095f4082be41165d1078204d1696c2d9baade3de3202d0/setproctitle-1.3.7-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2906b6c7959cdb75f46159bf0acd8cc9906cf1361c9e1ded0d065fe8f9039629", size = 32932, upload-time = "2025-09-05T12:49:39.271Z" }, + { url = "https://files.pythonhosted.org/packages/50/22/cee06af4ffcfb0e8aba047bd44f5262e644199ae7527ae2c1f672b86495c/setproctitle-1.3.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6915964a6dda07920a1159321dcd6d94fc7fc526f815ca08a8063aeca3c204f1", size = 33736, upload-time = "2025-09-05T12:49:40.565Z" }, + { url = "https://files.pythonhosted.org/packages/5c/00/a5949a8bb06ef5e7df214fc393bb2fb6aedf0479b17214e57750dfdd0f24/setproctitle-1.3.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cff72899861c765bd4021d1ff1c68d60edc129711a2fdba77f9cb69ef726a8b6", size = 35605, upload-time = "2025-09-05T12:49:42.362Z" }, + { url = "https://files.pythonhosted.org/packages/b0/3a/50caca532a9343828e3bf5778c7a84d6c737a249b1796d50dd680290594d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b7cb05bd446687ff816a3aaaf831047fc4c364feff7ada94a66024f1367b448c", size = 33143, upload-time = "2025-09-05T12:49:43.515Z" }, + { url = "https://files.pythonhosted.org/packages/ca/14/b843a251296ce55e2e17c017d6b9f11ce0d3d070e9265de4ecad948b913d/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3a57b9a00de8cae7e2a1f7b9f0c2ac7b69372159e16a7708aa2f38f9e5cc987a", size = 34434, upload-time = "2025-09-05T12:49:45.31Z" }, + { url = "https://files.pythonhosted.org/packages/c8/b7/06145c238c0a6d2c4bc881f8be230bb9f36d2bf51aff7bddcb796d5eed67/setproctitle-1.3.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d8828b356114f6b308b04afe398ed93803d7fca4a955dd3abe84430e28d33739", size = 32795, upload-time = "2025-09-05T12:49:46.419Z" }, + { url = "https://files.pythonhosted.org/packages/87/ed/0a4f00315bc02510395b95eec3d4aa77c07192ee79f0baae77ea7b9603d8/setproctitle-1.3.7-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0564a936ea687cd24dffcea35903e2a20962aa6ac20e61dd3a207652401492dd", size = 33284, upload-time = "2025-09-05T12:49:52.741Z" }, + { url = "https://files.pythonhosted.org/packages/fc/e4/adf3c4c0a2173cb7920dc9df710bcc67e9bcdbf377e243b7a962dc31a51a/setproctitle-1.3.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a5d1cb3f81531f0eb40e13246b679a1bdb58762b170303463cb06ecc296f26d0", size = 34104, upload-time = "2025-09-05T12:49:54.416Z" }, + { url = "https://files.pythonhosted.org/packages/52/4f/6daf66394152756664257180439d37047aa9a1cfaa5e4f5ed35e93d1dc06/setproctitle-1.3.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a7d159e7345f343b44330cbba9194169b8590cb13dae940da47aa36a72aa9929", size = 35982, upload-time = "2025-09-05T12:49:56.295Z" }, + { url = "https://files.pythonhosted.org/packages/1b/62/f2c0595403cf915db031f346b0e3b2c0096050e90e0be658a64f44f4278a/setproctitle-1.3.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0b5074649797fd07c72ca1f6bff0406f4a42e1194faac03ecaab765ce605866f", size = 33150, upload-time = "2025-09-05T12:49:58.025Z" }, + { url = "https://files.pythonhosted.org/packages/a0/29/10dd41cde849fb2f9b626c846b7ea30c99c81a18a5037a45cc4ba33c19a7/setproctitle-1.3.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:61e96febced3f61b766115381d97a21a6265a0f29188a791f6df7ed777aef698", size = 34463, upload-time = "2025-09-05T12:49:59.424Z" }, + { url = "https://files.pythonhosted.org/packages/71/3c/cedd8eccfaf15fb73a2c20525b68c9477518917c9437737fa0fda91e378f/setproctitle-1.3.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:047138279f9463f06b858e579cc79580fbf7a04554d24e6bddf8fe5dddbe3d4c", size = 32848, upload-time = "2025-09-05T12:50:01.107Z" }, + { url = "https://files.pythonhosted.org/packages/52/09/f366eca0973cfbac1470068d1313fa3fe3de4a594683385204ec7f1c4101/setproctitle-1.3.7-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c18246d88e227a5b16248687514f95642505000442165f4b7db354d39d0e4c29", size = 34490, upload-time = "2025-09-05T12:50:04.948Z" }, + { url = "https://files.pythonhosted.org/packages/71/36/611fc2ed149fdea17c3677e1d0df30d8186eef9562acc248682b91312706/setproctitle-1.3.7-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7081f193dab22df2c36f9fc6d113f3793f83c27891af8fe30c64d89d9a37e152", size = 35267, upload-time = "2025-09-05T12:50:06.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/a4/64e77d0671446bd5a5554387b69e1efd915274686844bea733714c828813/setproctitle-1.3.7-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9cc9b901ce129350637426a89cfd650066a4adc6899e47822e2478a74023ff7c", size = 37376, upload-time = "2025-09-05T12:50:07.484Z" }, + { url = "https://files.pythonhosted.org/packages/89/bc/ad9c664fe524fb4a4b2d3663661a5c63453ce851736171e454fa2cdec35c/setproctitle-1.3.7-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:80e177eff2d1ec172188d0d7fd9694f8e43d3aab76a6f5f929bee7bf7894e98b", size = 33963, upload-time = "2025-09-05T12:50:09.056Z" }, + { url = "https://files.pythonhosted.org/packages/ab/01/a36de7caf2d90c4c28678da1466b47495cbbad43badb4e982d8db8167ed4/setproctitle-1.3.7-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:23e520776c445478a67ee71b2a3c1ffdafbe1f9f677239e03d7e2cc635954e18", size = 35550, upload-time = "2025-09-05T12:50:10.791Z" }, + { url = "https://files.pythonhosted.org/packages/dd/68/17e8aea0ed5ebc17fbf03ed2562bfab277c280e3625850c38d92a7b5fcd9/setproctitle-1.3.7-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5fa1953126a3b9bd47049d58c51b9dac72e78ed120459bd3aceb1bacee72357c", size = 33727, upload-time = "2025-09-05T12:50:12.032Z" }, + { url = "https://files.pythonhosted.org/packages/ab/26/8e3bb082992f19823d831f3d62a89409deb6092e72fc6940962983ffc94f/setproctitle-1.3.7-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fcb966a6c57cf07cc9448321a08f3be6b11b7635be502669bc1d8745115d7e7f", size = 33180, upload-time = "2025-09-05T12:50:20.395Z" }, + { url = "https://files.pythonhosted.org/packages/f1/af/ae692a20276d1159dd0cf77b0bcf92cbb954b965655eb4a69672099bb214/setproctitle-1.3.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:46178672599b940368d769474fe13ecef1b587d58bb438ea72b9987f74c56ea5", size = 34043, upload-time = "2025-09-05T12:50:22.454Z" }, + { url = "https://files.pythonhosted.org/packages/34/b2/6a092076324dd4dac1a6d38482bedebbff5cf34ef29f58585ec76e47bc9d/setproctitle-1.3.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7f9e9e3ff135cbcc3edd2f4cf29b139f4aca040d931573102742db70ff428c17", size = 35892, upload-time = "2025-09-05T12:50:23.937Z" }, + { url = "https://files.pythonhosted.org/packages/1c/1a/8836b9f28cee32859ac36c3df85aa03e1ff4598d23ea17ca2e96b5845a8f/setproctitle-1.3.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:14c7eba8d90c93b0e79c01f0bd92a37b61983c27d6d7d5a3b5defd599113d60e", size = 32898, upload-time = "2025-09-05T12:50:25.617Z" }, + { url = "https://files.pythonhosted.org/packages/ef/22/8fabdc24baf42defb599714799d8445fe3ae987ec425a26ec8e80ea38f8e/setproctitle-1.3.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:9e64e98077fb30b6cf98073d6c439cd91deb8ebbf8fc62d9dbf52bd38b0c6ac0", size = 34308, upload-time = "2025-09-05T12:50:26.827Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/b9bee9de6c8cdcb3b3a6cb0b3e773afdb86bbbc1665a3bfa424a4294fda2/setproctitle-1.3.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b91387cc0f02a00ac95dcd93f066242d3cca10ff9e6153de7ee07069c6f0f7c8", size = 32536, upload-time = "2025-09-05T12:50:28.5Z" }, + { url = "https://files.pythonhosted.org/packages/21/9c/980b01f50d51345dd513047e3ba9e96468134b9181319093e61db1c47188/setproctitle-1.3.7-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1403d2abfd32790b6369916e2313dffbe87d6b11dca5bbd898981bcde48e7a2b", size = 34744, upload-time = "2025-09-05T12:50:32.777Z" }, + { url = "https://files.pythonhosted.org/packages/86/b4/82cd0c86e6d1c4538e1a7eb908c7517721513b801dff4ba3f98ef816a240/setproctitle-1.3.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e7c5bfe4228ea22373e3025965d1a4116097e555ee3436044f5c954a5e63ac45", size = 35589, upload-time = "2025-09-05T12:50:34.13Z" }, + { url = "https://files.pythonhosted.org/packages/8a/4f/9f6b2a7417fd45673037554021c888b31247f7594ff4bd2239918c5cd6d0/setproctitle-1.3.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:585edf25e54e21a94ccb0fe81ad32b9196b69ebc4fc25f81da81fb8a50cca9e4", size = 37698, upload-time = "2025-09-05T12:50:35.524Z" }, + { url = "https://files.pythonhosted.org/packages/20/92/927b7d4744aac214d149c892cb5fa6dc6f49cfa040cb2b0a844acd63dcaf/setproctitle-1.3.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:96c38cdeef9036eb2724c2210e8d0b93224e709af68c435d46a4733a3675fee1", size = 34201, upload-time = "2025-09-05T12:50:36.697Z" }, + { url = "https://files.pythonhosted.org/packages/0a/0c/fd4901db5ba4b9d9013e62f61d9c18d52290497f956745cd3e91b0d80f90/setproctitle-1.3.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:45e3ef48350abb49cf937d0a8ba15e42cee1e5ae13ca41a77c66d1abc27a5070", size = 35801, upload-time = "2025-09-05T12:50:38.314Z" }, + { url = "https://files.pythonhosted.org/packages/e7/e3/54b496ac724e60e61cc3447f02690105901ca6d90da0377dffe49ff99fc7/setproctitle-1.3.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1fae595d032b30dab4d659bece20debd202229fce12b55abab978b7f30783d73", size = 33958, upload-time = "2025-09-05T12:50:39.841Z" }, + { url = "https://files.pythonhosted.org/packages/73/02/b9eadc226195dcfa90eed37afe56b5dd6fa2f0e5220ab8b7867b8862b926/setproctitle-1.3.7-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f1704c9e041f2b1dc38f5be4552e141e1432fba3dd52c72eeffd5bc2db04dc65", size = 14286, upload-time = "2025-09-05T12:51:22.61Z" }, +] + [[package]] name = "setuptools" -version = "82.0.0" +version = "80.10.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/f3/748f4d6f65d1756b9ae577f329c951cda23fb900e4de9f70900ced962085/setuptools-82.0.0.tar.gz", hash = "sha256:22e0a2d69474c6ae4feb01951cb69d515ed23728cf96d05513d36e42b62b37cb", size = 1144893, upload-time = "2026-02-08T15:08:40.206Z" } +sdist = { url = "https://files.pythonhosted.org/packages/76/95/faf61eb8363f26aa7e1d762267a8d602a1b26d4f3a1e758e92cb3cb8b054/setuptools-80.10.2.tar.gz", hash = "sha256:8b0e9d10c784bf7d262c4e5ec5d4ec94127ce206e8738f29a437945fbc219b70", size = 1200343, upload-time = "2026-01-25T22:38:17.252Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/c6/76dc613121b793286a3f91621d7b75a2b493e0390ddca50f11993eadf192/setuptools-82.0.0-py3-none-any.whl", hash = "sha256:70b18734b607bd1da571d097d236cfcfacaf01de45717d59e6e04b96877532e0", size = 1003468, upload-time = "2026-02-08T15:08:38.723Z" }, + { url = "https://files.pythonhosted.org/packages/94/b8/f1f62a5e3c0ad2ff1d189590bfa4c46b4f3b6e49cef6f26c6ee4e575394d/setuptools-80.10.2-py3-none-any.whl", hash = "sha256:95b30ddfb717250edb492926c92b5221f7ef3fbcc2b07579bcd4a27da21d0173", size = 1064234, upload-time = "2026-01-25T22:38:15.216Z" }, ] [[package]] @@ -4094,6 +6013,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, ] +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + [[package]] name = "soupsieve" version = "2.8.3" @@ -4105,7 +6033,7 @@ wheels = [ [[package]] name = "sqlfluff" -version = "4.2.1" +version = "4.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "chardet" }, @@ -4120,9 +6048,9 @@ dependencies = [ { name = "tblib" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b4/a1/3430aebc4fae35d7e466e793b5da2f36c2245af092311520dc1d3d3146d6/sqlfluff-4.2.1.tar.gz", hash = "sha256:32f43fbf6721e57f1a5a87d71df0d94b84ecba6ed65727266c7fa60991110fb9", size = 1013384, upload-time = "2026-05-14T21:15:37.161Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/06/c675066797f91a77d456bc15375a0a8c9b4765f1e66db639655772a928da/sqlfluff-4.3.0.tar.gz", hash = "sha256:aa647b3721112f1aca581efe8eaa815a332ae214610a0946592f98a19ef1e8cb", size = 1068771, upload-time = "2026-08-07T09:01:19.317Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/55/8830f3204939cc965c72680bfae99b0f3fd6c16bffdaf79372ad0a3d1ca6/sqlfluff-4.2.1-py3-none-any.whl", hash = "sha256:ea84f196c41f45df40a851b0881cb3fbb660570e07acbbfd304ff4e9b893424d", size = 1002493, upload-time = "2026-05-14T21:15:35.582Z" }, + { url = "https://files.pythonhosted.org/packages/79/be/0be38ce154b9d798bb00b5ce65a3ad0e1ae2bc9419fcd8e3c773c2b80c70/sqlfluff-4.3.0-py3-none-any.whl", hash = "sha256:582e12af5101bd1c5bc120cb4a6942dadcfe5e5b1e01406f220dd08f5b3f31dd", size = 1051485, upload-time = "2026-08-07T09:01:17.139Z" }, ] [[package]] @@ -4165,6 +6093,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] +[[package]] +name = "supervisor" +version = "4.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/b5/37e7a3706de436a8a2d75334711dad1afb4ddffab09f25e31d89e467542f/supervisor-4.3.0.tar.gz", hash = "sha256:4a2bf149adf42997e1bb44b70c43b613275ec9852c3edacca86a9166b27e945e", size = 468912, upload-time = "2025-08-23T18:25:02.418Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/65/5e726c372da8a5e35022a94388b12252710aad0c2351699c3d76ae8dba78/supervisor-4.3.0-py2.py3-none-any.whl", hash = "sha256:0bcb763fddafba410f35cbde226aa7f8514b9fb82eb05a0c85f6588d1c13f8db", size = 320736, upload-time = "2025-08-23T18:25:00.767Z" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "tabulate" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/46/58/8c37dea7bbf769b20d58e7ace7e5edfe65b849442b00ffcdd56be88697c6/tabulate-0.10.0.tar.gz", hash = "sha256:e2cfde8f79420f6deeffdeda9aaec3b6bc5abce947655d17ac662b126e48a60d", size = 91754, upload-time = "2026-03-04T18:55:34.402Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/55/db07de81b5c630da5cbf5c7df646580ca26dfaefa593667fc6f2fe016d2e/tabulate-0.10.0-py3-none-any.whl", hash = "sha256:f0b0622e567335c8fabaaa659f1b33bcb6ddfe2e496071b743aa113f8774f2d3", size = 39814, upload-time = "2026-03-04T18:55:31.284Z" }, +] + [[package]] name = "tblib" version = "3.2.2" @@ -4180,7 +6138,7 @@ version = "0.18.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ptyprocess", marker = "os_name != 'nt'" }, - { name = "pywinpty", marker = "os_name == 'nt'" }, + { name = "pywinpty", marker = "os_name == 'nt' and sys_platform != 'darwin'" }, { name = "tornado" }, ] sdist = { url = "https://files.pythonhosted.org/packages/8a/11/965c6fd8e5cc254f1fe142d547387da17a8ebfd75a3455f637c663fb38a0/terminado-0.18.1.tar.gz", hash = "sha256:de09f2c4b85de4765f7714688fff57d3e75bad1f909b589fde880460c753fd2e", size = 32701, upload-time = "2024-03-12T14:34:39.026Z" } @@ -4242,6 +6200,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/af/df/c7891ef9d2712ad774777271d39fdef63941ffba0a9d59b7ad1fd2765e57/tiktoken-0.12.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f61c0aea5565ac82e2ec50a05e02a6c44734e91b51c10510b084ea1b8e633a71", size = 920667, upload-time = "2025-10-06T20:22:34.444Z" }, ] +[[package]] +name = "tilelang" +version = "0.1.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi" }, + { name = "cloudpickle" }, + { name = "ml-dtypes" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "psutil" }, + { name = "setuptools", marker = "python_version < '0'" }, + { name = "torch" }, + { name = "torch-c-dlpack-ext", marker = "python_full_version < '3.14'" }, + { name = "tqdm" }, + { name = "typing-extensions" }, + { name = "z3-solver" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/56/70/5051f65821baa30a3d61fc48f8ba10c776490315e8c90f82559b92089756/tilelang-0.1.9.tar.gz", hash = "sha256:287f727c913bb648fcf6c1968809ba3390e55eeed257a5c6bb9a80bc05966af4", size = 93395292, upload-time = "2026-04-22T09:19:11.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/8a/1cbeee79d62abaa02441c2d00621554e41aa62dbf3b94a4feb3867184b01/tilelang-0.1.9-cp38-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bbccfe9035aed775ffafb6dc25a5994504b24e2c5d95d0f39643edfafa7bf12", size = 45419374, upload-time = "2026-04-22T09:15:56.014Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a7/f4bfb86f87e107703146e703204cec2c0eae2492b633e0052b0ace3febb6/tilelang-0.1.9-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:77ab0ee2f40f66ea015b6b21426d482751e28cbc635ef9d1198cbd6502454a7c", size = 42110365, upload-time = "2026-04-22T09:17:18.292Z" }, +] + [[package]] name = "tinycss2" version = "1.4.0" @@ -4254,6 +6236,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e6/34/ebdc18bae6aa14fbee1a08b63c015c72b64868ff7dae68808ab500c492e2/tinycss2-1.4.0-py3-none-any.whl", hash = "sha256:3a49cf47b7675da0b15d0c6e1df8df4ebd96e9394bb905a5775adb0d884c5289", size = 26610, upload-time = "2024-10-24T14:58:28.029Z" }, ] +[[package]] +name = "tokenizers" +version = "0.22.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/84/7990e799f1309a8b87af6b948f31edaa12a3ed22d11b352eaf4f4b2e5753/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2249487018adec45d6e3554c71d46eb39fa8ea67156c640f7513eb26f318cec7", size = 3290736, upload-time = "2026-01-05T10:40:32.165Z" }, + { url = "https://files.pythonhosted.org/packages/78/59/09d0d9ba94dcd5f4f1368d4858d24546b4bdc0231c2354aa31d6199f0399/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:25b85325d0815e86e0bac263506dd114578953b7b53d7de09a6485e4a160a7dd", size = 3168835, upload-time = "2026-01-05T10:40:38.847Z" }, + { url = "https://files.pythonhosted.org/packages/47/50/b3ebb4243e7160bda8d34b731e54dd8ab8b133e50775872e7a434e524c28/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bfb88f22a209ff7b40a576d5324bf8286b519d7358663db21d6246fb17eea2d5", size = 3521673, upload-time = "2026-01-05T10:40:56.614Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fa/89f4cb9e08df770b57adb96f8cbb7e22695a4cb6c2bd5f0c4f0ebcf33b66/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c774b1276f71e1ef716e5486f21e76333464f47bece56bbd554485982a9e03e", size = 3724818, upload-time = "2026-01-05T10:40:44.507Z" }, + { url = "https://files.pythonhosted.org/packages/64/04/ca2363f0bfbe3b3d36e95bf67e56a4c88c8e3362b658e616d1ac185d47f2/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:df6c4265b289083bf710dff49bc51ef252f9d5be33a45ee2bed151114a56207b", size = 3379195, upload-time = "2026-01-05T10:40:51.139Z" }, + { url = "https://files.pythonhosted.org/packages/2e/76/932be4b50ef6ccedf9d3c6639b056a967a86258c6d9200643f01269211ca/tokenizers-0.22.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:369cc9fc8cc10cb24143873a0d95438bb8ee257bb80c71989e3ee290e8d72c67", size = 3274982, upload-time = "2026-01-05T10:40:58.331Z" }, + { url = "https://files.pythonhosted.org/packages/1d/28/5f9f5a4cc211b69e89420980e483831bcc29dade307955cc9dc858a40f01/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:29c30b83d8dcd061078b05ae0cb94d3c710555fbb44861139f9f83dcca3dc3e4", size = 9478245, upload-time = "2026-01-05T10:41:04.053Z" }, + { url = "https://files.pythonhosted.org/packages/6c/fb/66e2da4704d6aadebf8cb39f1d6d1957df667ab24cff2326b77cda0dcb85/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:37ae80a28c1d3265bb1f22464c856bd23c02a05bb211e56d0c5301a435be6c1a", size = 9560069, upload-time = "2026-01-05T10:45:10.673Z" }, + { url = "https://files.pythonhosted.org/packages/16/04/fed398b05caa87ce9b1a1bb5166645e38196081b225059a6edaff6440fac/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:791135ee325f2336f498590eb2f11dc5c295232f288e75c99a36c5dbce63088a", size = 9899263, upload-time = "2026-01-05T10:45:12.559Z" }, + { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, +] + [[package]] name = "tomli" version = "2.4.0" @@ -4308,6 +6311,104 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/d1/136eb2cb77520a31e1f64cbae9d33ec6df0d78bdf4160398e86eec8a8754/tomli-2.4.0-py3-none-any.whl", hash = "sha256:1f776e7d669ebceb01dee46484485f43a4048746235e683bcdffacdf1fb4785a", size = 14477, upload-time = "2026-01-11T11:22:37.446Z" }, ] +[[package]] +name = "torch" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"] }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cudnn-cu13" }, + { name = "nvidia-cusparselt-cu13" }, + { name = "nvidia-nccl-cu13" }, + { name = "nvidia-nvshmem-cu13" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/03/acea680005f098f79fd70c1d9d5ccc0cb4296ec2af539a0450108232fc0c/torch-2.11.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d91aac77f24082809d2c5a93f52a5f085032740a1ebc9252a7b052ef5a4fddc6", size = 419718178, upload-time = "2026-03-23T18:10:46.675Z" }, + { url = "https://files.pythonhosted.org/packages/8c/8b/d7be22fbec9ffee6cff31a39f8750d4b3a65d349a286cf4aec74c2375662/torch-2.11.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7aa2f9bbc6d4595ba72138026b2074be1233186150e9292865e04b7a63b8c67a", size = 530604548, upload-time = "2026-03-23T18:10:03.569Z" }, + { url = "https://files.pythonhosted.org/packages/13/16/42e5915ebe4868caa6bac83a8ed59db57f12e9a61b7d749d584776ed53d5/torch-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f99924682ef0aa6a4ab3b1b76f40dc6e273fca09f367d15a524266db100a723f", size = 419731115, upload-time = "2026-03-23T18:11:06.944Z" }, + { url = "https://files.pythonhosted.org/packages/1a/c9/82638ef24d7877510f83baf821f5619a61b45568ce21c0a87a91576510aa/torch-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0f68f4ac6d95d12e896c3b7a912b5871619542ec54d3649cf48cc1edd4dd2756", size = 530712279, upload-time = "2026-03-23T18:10:31.481Z" }, + { url = "https://files.pythonhosted.org/packages/32/d1/8ed2173589cbfe744ed54e5a73efc107c0085ba5777ee93a5f4c1ab90553/torch-2.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:63a68fa59de8f87acc7e85a5478bb2dddbb3392b7593ec3e78827c793c4b73fd", size = 419732382, upload-time = "2026-03-23T18:08:30.835Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e1/b73f7c575a4b8f87a5928f50a1e35416b5e27295d8be9397d5293e7e8d4c/torch-2.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cc89b9b173d9adfab59fd227f0ab5e5516d9a52b658ae41d64e59d2e55a418db", size = 530711509, upload-time = "2026-03-23T18:08:47.213Z" }, + { url = "https://files.pythonhosted.org/packages/6d/6c/56bfb37073e7136e6dd86bfc6af7339946dd684e0ecf2155ac0eee687ae1/torch-2.11.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:2658f34ce7e2dabf4ec73b45e2ca68aedad7a5be87ea756ad656eaf32bf1e1ea", size = 419732324, upload-time = "2026-03-23T18:09:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/07/f4/1b666b6d61d3394cca306ea543ed03a64aad0a201b6cd159f1d41010aeb1/torch-2.11.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:98bb213c3084cfe176302949bdc360074b18a9da7ab59ef2edc9d9f742504778", size = 530596026, upload-time = "2026-03-23T18:09:20.842Z" }, + { url = "https://files.pythonhosted.org/packages/c7/86/7cd7c66cb9cec6be330fff36db5bd0eef386d80c031b581ec81be1d4b26c/torch-2.11.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:2bb3cc54bd0dea126b0060bb1ec9de0f9c7f7342d93d436646516b0330cd5be7", size = 419749385, upload-time = "2026-03-23T18:07:33.77Z" }, + { url = "https://files.pythonhosted.org/packages/47/e8/b98ca2d39b2e0e4730c0ee52537e488e7008025bc77ca89552ff91021f7c/torch-2.11.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4dc8b3809469b6c30b411bb8c4cad3828efd26236153d9beb6a3ec500f211a60", size = 530716756, upload-time = "2026-03-23T18:07:50.02Z" }, + { url = "https://files.pythonhosted.org/packages/fd/66/54a56a4a6ceaffb567231994a9745821d3af922a854ed33b0b3a278e0a99/torch-2.11.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:ab9a8482f475f9ba20e12db84b0e55e2f58784bdca43a854a6ccd3fd4b9f75e6", size = 419735835, upload-time = "2026-03-23T18:07:18.974Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e7/0b6665f533aa9e337662dc190425abc0af1fe3234088f4454c52393ded61/torch-2.11.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:563ed3d25542d7e7bbc5b235ccfacfeb97fb470c7fee257eae599adb8005c8a2", size = 530613405, upload-time = "2026-03-23T18:08:07.014Z" }, +] + +[[package]] +name = "torch-c-dlpack-ext" +version = "0.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "torch" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/37/de/921b6491efce5c389a5ef9bbed3d2d6660005840dae488124173180859ab/torch_c_dlpack_ext-0.1.5.tar.gz", hash = "sha256:d06f0357d575d22a168cc77acb9020fc4bae30968ceb6718a055dcbe92bacabe", size = 12913, upload-time = "2026-01-12T11:25:08.484Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/e1/64e1e579d107064785549e70758e38a42376ab7e73d86897ed4beab10e74/torch_c_dlpack_ext-0.1.5-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fba674110e1fab0b176bb5a28223e157db65c90767d4ba74abdbee9f537b0e9d", size = 440949, upload-time = "2026-01-12T11:24:39.716Z" }, + { url = "https://files.pythonhosted.org/packages/64/5c/3e1382a620824f92920ab3fae132d8fb4e85898284c99e0c6a7764e452ce/torch_c_dlpack_ext-0.1.5-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3448c4f0d64104d0b2e58080a7efa72304a04960c18f338024b80b13cd3eca26", size = 897768, upload-time = "2026-01-12T11:24:41.209Z" }, + { url = "https://files.pythonhosted.org/packages/87/06/8d760997307a5c3be4384424667bf31aae0a42060838c532c7d846516175/torch_c_dlpack_ext-0.1.5-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3562ee411258676f9c38b8ad39306d1c8d027b6a86f6a87c920d2d009a9d1510", size = 443069, upload-time = "2026-01-12T11:24:45.451Z" }, + { url = "https://files.pythonhosted.org/packages/e2/79/a914539b4785f3e44f891aa012a886edb8bc10fe081c440981c57543ce21/torch_c_dlpack_ext-0.1.5-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e6f9da4bb9af70e27facc777458be62e10dbbbddda7672d16138db0553c5a524", size = 897846, upload-time = "2026-01-12T11:24:48.168Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ec/faf10be09a5812b1c5ec9922b53fb5def5fc4080b81a653b9347bb169ebb/torch_c_dlpack_ext-0.1.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49f1e99d13c64e22dac0a34a1560e9e5a398a49a9fa81df83053e04fde6ec5bd", size = 443798, upload-time = "2026-01-12T11:24:52.754Z" }, + { url = "https://files.pythonhosted.org/packages/2d/68/f434b48700f3e04f33882f54d8d3910327b935f55e14ec49da7d607bf470/torch_c_dlpack_ext-0.1.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:debe62e5ef93e631065d6b9f6e60d3d39bae6b89fa1b25d9523f40b3efbf8aba", size = 755004, upload-time = "2026-01-12T11:24:54.004Z" }, + { url = "https://files.pythonhosted.org/packages/20/62/11c05b99f69aa5152bca0313e0dfa6d125a020cf890dc888ef009aa7891c/torch_c_dlpack_ext-0.1.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a58fdf45fb0bda7bc459632cec891570f31c11636d5851c825cf308ec8b73c2", size = 163825, upload-time = "2026-01-12T11:24:59.474Z" }, + { url = "https://files.pythonhosted.org/packages/15/b5/be613cd8e71c9982bd07af530f86c5a7f30df7831d14cec5414857af7149/torch_c_dlpack_ext-0.1.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b985a324c68241cf83a9474b28015524b66775b12a91930dd4c0760aa628d01", size = 171740, upload-time = "2026-01-12T11:25:00.776Z" }, +] + +[[package]] +name = "torchaudio" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/f9/6f7ebe071b44592c85269762b55b63ab0a091b5f479f73544738f7564a1e/torchaudio-2.11.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:73dab4841f94d888bc7c2aed7b5547c643edc974306919fe1adfb65d57cccf4b", size = 1626527, upload-time = "2026-03-23T18:13:39.011Z" }, + { url = "https://files.pythonhosted.org/packages/ac/70/17408e0d154d0c894537a88dcbadc48e8ad3b6e1ef4a1dabda5d40245ee0/torchaudio-2.11.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1a07ec72fd6f26a588c39b5f029e0130d16bb40bc4221635580bf8fb18fcbc80", size = 1771930, upload-time = "2026-03-23T18:13:37.963Z" }, + { url = "https://files.pythonhosted.org/packages/78/28/c7adc053039f286c2aca0038b766cbe3294e66fec6b29a820e95128f9ede/torchaudio-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bc653defca1c16154398517a1adc98d0fb7f1dd08e58ced217558d213c2c6e29", size = 1626670, upload-time = "2026-03-23T18:13:42.162Z" }, + { url = "https://files.pythonhosted.org/packages/88/d8/d6d0f896e064aa67377484efef4911cdcc07bce2929474e1417cc0af18c2/torchaudio-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6503c0bdb29daf2e6281bb70ea2dfe2c3553b782b619eb5d73bdadd8a3f7cecf", size = 1771992, upload-time = "2026-03-23T18:13:33.188Z" }, + { url = "https://files.pythonhosted.org/packages/85/70/249c1498ebdad3e7752866635ec0855fc0dcf898beccda5a9d2b9df8e4d0/torchaudio-2.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:b034d7672f1c415434f48ef17807f2cce47f29e8795338c751d4e596c9fbe8b5", size = 1618523, upload-time = "2026-03-23T18:13:15.703Z" }, + { url = "https://files.pythonhosted.org/packages/4f/98/be13fe35d9aa5c26381c0e453c828a789d15c007f8f7d08c95341d19974d/torchaudio-2.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1c1101c1243ef0e4063ec63298977e2d3655c15cf88d9eb0a1bd4fe2db9f47ea", size = 1771992, upload-time = "2026-03-23T18:13:35.343Z" }, + { url = "https://files.pythonhosted.org/packages/06/95/1ad1507482e7263e556709a3f5f87fecd375a0742cdaf238806c8e72eaad/torchaudio-2.11.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:9fe3083c62e035646483a14e180d33561bdc2eed436c9ab1259c137fb7120b4a", size = 1618546, upload-time = "2026-03-23T18:13:29.686Z" }, + { url = "https://files.pythonhosted.org/packages/98/4c/480328ba07487eb9890406720304d0d460dd7a6a64098614f5aa53b662ca/torchaudio-2.11.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:13cff988697ccbad539987599f9dc672f40c417bed67570b365e4e5002bbd096", size = 1771991, upload-time = "2026-03-23T18:13:30.843Z" }, + { url = "https://files.pythonhosted.org/packages/5c/54/f414d7b92dd0b3094a2409c95a97bd6c49aa0620da722a0e55462f9bd9cb/torchaudio-2.11.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:79fb3cb99169fd41bd9719647261402a164da0d105a4d81f42a3260844ec5e79", size = 1618527, upload-time = "2026-03-23T18:13:26.68Z" }, + { url = "https://files.pythonhosted.org/packages/a8/a8/bf2e1f6ce24c990192400ae49b4acc1a0d0295b6c6a06bceecdc46ce08de/torchaudio-2.11.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:00e9f71ab9c656f0abdb40c515bd65d4658ab0ad380dee27a2efd7d51dabd3d6", size = 1771995, upload-time = "2026-03-23T18:13:23.373Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a0/62a5842062f739239691f2e57523e0570dd06704ad987755f7644a3afa23/torchaudio-2.11.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:1be3767064364ae82705bdf2b15c1e8b41fea82c4cd04d47428a8684b634b6ed", size = 1618552, upload-time = "2026-03-23T18:13:21.09Z" }, + { url = "https://files.pythonhosted.org/packages/6d/89/c293d818f9f899db93bf291b42401c05ae29acfb2e53d5341c30ea703e62/torchaudio-2.11.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:67f6edac29ed004652c11db5c19d9debb5d835695930574f564efc8bdd061bba", size = 1771986, upload-time = "2026-03-23T18:13:22.153Z" }, +] + +[[package]] +name = "torchvision" +version = "0.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "pillow" }, + { name = "torch" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/bf/21b899792b08cae7a298551c68398a79e333697479ed311b3b067aab4bdc/torchvision-0.26.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1c55dc8affbcc0eb2060fbabbe996ae9e5839b24bb6419777f17848945a411b1", size = 7767527, upload-time = "2026-03-23T18:12:44.348Z" }, + { url = "https://files.pythonhosted.org/packages/9a/45/57bbf9e216850d065e66dd31a50f57424b607f1d878ab8956e56a1f4e36b/torchvision-0.26.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:fd10b5f994c210f4f6d6761cf686f82d748554adf486cb0979770c3252868c8f", size = 7519925, upload-time = "2026-03-23T18:12:53.283Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ec/5c31c92c08b65662fe9604a4067ae8232582805949f11ddc042cebe818ed/torchvision-0.26.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:406557718e62fdf10f5706e88d8a5ec000f872da913bf629aab9297622585547", size = 7767944, upload-time = "2026-03-23T18:12:42.805Z" }, + { url = "https://files.pythonhosted.org/packages/f5/d8/cb6ccda1a1f35a6597645818641701207b3e8e13553e75fce5d86bac74b2/torchvision-0.26.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d61a5abb6b42a0c0c311996c2ac4b83a94418a97182c83b055a2a4ae985e05aa", size = 7522205, upload-time = "2026-03-23T18:12:54.654Z" }, + { url = "https://files.pythonhosted.org/packages/e6/81/0b3e58d1478c660a5af4268713486b2df7203f35abd9195fea87348a5178/torchvision-0.26.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a39c7a26538c41fda453f9a9692b5ff9b35a5437db1d94f3027f6f509c160eac", size = 7727494, upload-time = "2026-03-23T18:12:46.062Z" }, + { url = "https://files.pythonhosted.org/packages/b6/dc/d9ab5d29115aa05e12e30f1397a3eeae1d88a511241dc3bce48dc4342675/torchvision-0.26.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:b7e6213620bbf97742e5f79832f9e9d769e6cf0f744c5b53dad80b76db633691", size = 7521747, upload-time = "2026-03-23T18:12:36.815Z" }, + { url = "https://files.pythonhosted.org/packages/71/e2/7a89096e6cf2f3336353b5338ba925e0addf9d8601920340e6bdf47e8eb3/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:3daf9cc149cf3cdcbd4df9c59dae69ffca86c6823250442c3bbfd63fc2e26c61", size = 7728679, upload-time = "2026-03-23T18:12:26.196Z" }, + { url = "https://files.pythonhosted.org/packages/69/1d/4e1eebc17d18ce080a11dcf3df3f8f717f0efdfa00983f06e8ba79259f61/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:82c3965eca27e86a316e31e4c3e5a16d353e0bcbe0ef8efa2e66502c54493c4b", size = 7609138, upload-time = "2026-03-23T18:12:35.327Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ac/48f28ffd227991f2e14f4392dde7e8dc14352bb9428c1ef4a4bbf5f7ed85/torchvision-0.26.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:9a904f2131cbfadab4df828088a9f66291ad33f49ff853872aed1f86848ef776", size = 7727777, upload-time = "2026-03-23T18:12:22.549Z" }, + { url = "https://files.pythonhosted.org/packages/a4/21/a2266f7f1b0e58e624ff15fd6f01041f59182c49551ece0db9a183071329/torchvision-0.26.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0f3e572efe62ad645017ea847e0b5e4f2f638d4e39f05bc011d1eb9ac68d4806", size = 7522174, upload-time = "2026-03-23T18:12:29.565Z" }, + { url = "https://files.pythonhosted.org/packages/d2/6a/18a582fe3c5ee26f49b5c9fb21ad8016b4d1c06d10178894a58653946fda/torchvision-0.26.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:7058c5878262937e876f20c25867b33724586aa4499e2853b2d52b99a5e51953", size = 7729089, upload-time = "2026-03-23T18:12:31.394Z" }, + { url = "https://files.pythonhosted.org/packages/c5/9b/f7e119b59499edc00c55c03adc9ec3bd96144d9b81c46852c431f9c64a9a/torchvision-0.26.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:8008474855623c6ba52876589dc52df0aa66e518c25eca841445348e5f79844c", size = 7522704, upload-time = "2026-03-23T18:12:20.301Z" }, +] + [[package]] name = "tornado" version = "6.5.5" @@ -4346,6 +6447,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359, upload-time = "2024-04-19T11:11:46.763Z" }, ] +[[package]] +name = "transformers" +version = "5.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/3f/d89353267d511e18f137dfd7769d07837350c11b88408ce1dfe2e93e56c7/transformers-5.15.0.tar.gz", hash = "sha256:bbf98f57b2ddd7c4ecbccfa2c0069017aa6fd01cc204bd50cbc0eeadcf2a13b8", size = 9377983, upload-time = "2026-08-10T10:27:23.261Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/43/81355710a4c84e9420e11a86d41a5364deb561f2ef36dfdf254a07371bbb/transformers-5.15.0-py3-none-any.whl", hash = "sha256:d7f007736f67749ae9490c4f8cb5d30b452ae2d68c8675e50ba8d63ea7feb107", size = 11749280, upload-time = "2026-08-10T10:27:20.416Z" }, +] + +[[package]] +name = "triton" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/2c/96f92f3c60387e14cc45aed49487f3486f89ea27106c1b1376913c62abe4/triton-3.6.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49df5ef37379c0c2b5c0012286f80174fcf0e073e5ade1ca9a86c36814553651", size = 176081190, upload-time = "2026-01-20T16:16:00.523Z" }, + { url = "https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3", size = 188214640, upload-time = "2026-01-20T16:00:35.869Z" }, + { url = "https://files.pythonhosted.org/packages/17/5d/08201db32823bdf77a0e2b9039540080b2e5c23a20706ddba942924ebcd6/triton-3.6.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4", size = 176128243, upload-time = "2026-01-20T16:16:07.857Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a8/cdf8b3e4c98132f965f88c2313a4b493266832ad47fb52f23d14d4f86bb5/triton-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca", size = 188266850, upload-time = "2026-01-20T16:00:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/3c/12/34d71b350e89a204c2c7777a9bba0dcf2f19a5bfdd70b57c4dbc5ffd7154/triton-3.6.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448e02fe6dc898e9e5aa89cf0ee5c371e99df5aa5e8ad976a80b93334f3494fd", size = 176133521, upload-time = "2026-01-20T16:16:13.321Z" }, + { url = "https://files.pythonhosted.org/packages/f9/0b/37d991d8c130ce81a8728ae3c25b6e60935838e9be1b58791f5997b24a54/triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c7f76c6e72d2ef08df639e3d0d30729112f47a56b0c81672edc05ee5116ac9", size = 188289450, upload-time = "2026-01-20T16:00:49.136Z" }, + { url = "https://files.pythonhosted.org/packages/ce/4e/41b0c8033b503fd3cfcd12392cdd256945026a91ff02452bef40ec34bee7/triton-3.6.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1722e172d34e32abc3eb7711d0025bb69d7959ebea84e3b7f7a341cd7ed694d6", size = 176276087, upload-time = "2026-01-20T16:16:18.989Z" }, + { url = "https://files.pythonhosted.org/packages/35/f8/9c66bfc55361ec6d0e4040a0337fb5924ceb23de4648b8a81ae9d33b2b38/triton-3.6.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d002e07d7180fd65e622134fbd980c9a3d4211fb85224b56a0a0efbd422ab72f", size = 188400296, upload-time = "2026-01-20T16:00:56.042Z" }, + { url = "https://files.pythonhosted.org/packages/49/55/5ecf0dcaa0f2fbbd4420f7ef227ee3cb172e91e5fede9d0ecaddc43363b4/triton-3.6.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5523241e7d1abca00f1d240949eebdd7c673b005edbbce0aca95b8191f1d43", size = 176138577, upload-time = "2026-01-20T16:16:25.426Z" }, + { url = "https://files.pythonhosted.org/packages/df/3d/9e7eee57b37c80cec63322c0231bb6da3cfe535a91d7a4d64896fcb89357/triton-3.6.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a17a5d5985f0ac494ed8a8e54568f092f7057ef60e1b0fa09d3fd1512064e803", size = 188273063, upload-time = "2026-01-20T16:01:07.278Z" }, + { url = "https://files.pythonhosted.org/packages/48/db/56ee649cab5eaff4757541325aca81f52d02d4a7cd3506776cad2451e060/triton-3.6.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b3a97e8ed304dfa9bd23bb41ca04cdf6b2e617d5e782a8653d616037a5d537d", size = 176274804, upload-time = "2026-01-20T16:16:31.528Z" }, + { url = "https://files.pythonhosted.org/packages/f6/56/6113c23ff46c00aae423333eb58b3e60bdfe9179d542781955a5e1514cb3/triton-3.6.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46bd1c1af4b6704e554cad2eeb3b0a6513a980d470ccfa63189737340c7746a7", size = 188397994, upload-time = "2026-01-20T16:01:14.236Z" }, +] + +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "ty" version = "0.0.69" @@ -4386,18 +6536,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7a/ed/d6fca788b51d0d4640c4bc82d0e85bad4b49809bca36bf4af01b4dcb66a7/typer-0.23.0-py3-none-any.whl", hash = "sha256:79f4bc262b6c37872091072a3cb7cb6d7d79ee98c0c658b4364bdcde3c42c913", size = 56668, upload-time = "2026-02-11T15:22:21.075Z" }, ] -[[package]] -name = "typer-slim" -version = "0.23.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typer" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/8a/881cfd399a119db89619dc1b93d36e2fb6720ddb112bceff41203f1abd72/typer_slim-0.23.0.tar.gz", hash = "sha256:be8b60243df27cfee444c6db1b10a85f4f3e54d940574f31a996f78aa35a8254", size = 4773, upload-time = "2026-02-11T15:22:19.106Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/3e/ba3a222c80ee070d9497ece3e1fe77253c142925dd4c90f04278aac0a9eb/typer_slim-0.23.0-py3-none-any.whl", hash = "sha256:1d693daf22d998a7b1edab8413cdcb8af07254154ce3956c1664dc11b01e2f8b", size = 3399, upload-time = "2026-02-11T15:22:17.792Z" }, -] - [[package]] name = "typing-extensions" version = "4.15.0" @@ -4459,6 +6597,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3d/d8/2083a1daa7439a66f3a48589a57d576aa117726762618f6bb09fe3798796/uvicorn-0.40.0-py3-none-any.whl", hash = "sha256:c6c8f55bc8bf13eb6fa9ff87ad62308bbbc33d0b67f84293151efe87e0d5f2ee", size = 68502, upload-time = "2025-12-21T14:16:21.041Z" }, ] +[package.optional-dependencies] +standard = [ + { name = "colorama", marker = "python_version < '0'" }, + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy'" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + +[[package]] +name = "uvloop" +version = "0.22.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/86/94/9fb7fad2f824d25f8ecac0d70b94d0d48107ad5ece03769a9c543444f78a/uvloop-0.22.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:53c85520781d84a4b8b230e24a5af5b0778efdb39142b424990ff1ef7c48ba21", size = 3753819, upload-time = "2025-10-16T22:16:23.903Z" }, + { url = "https://files.pythonhosted.org/packages/74/4f/256aca690709e9b008b7108bc85fba619a2bc37c6d80743d18abad16ee09/uvloop-0.22.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56a2d1fae65fd82197cb8c53c367310b3eabe1bbb9fb5a04d28e3e3520e4f702", size = 3804529, upload-time = "2025-10-16T22:16:25.246Z" }, + { url = "https://files.pythonhosted.org/packages/7f/74/03c05ae4737e871923d21a76fe28b6aad57f5c03b6e6bfcfa5ad616013e4/uvloop-0.22.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:40631b049d5972c6755b06d0bfe8233b1bd9a8a6392d9d1c45c10b6f9e9b2733", size = 3621267, upload-time = "2025-10-16T22:16:26.819Z" }, + { url = "https://files.pythonhosted.org/packages/75/be/f8e590fe61d18b4a92070905497aec4c0e64ae1761498cad09023f3f4b3e/uvloop-0.22.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:535cc37b3a04f6cd2c1ef65fa1d370c9a35b6695df735fcff5427323f2cd5473", size = 3723105, upload-time = "2025-10-16T22:16:28.252Z" }, + { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, + { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, + { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, + { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, + { url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562, upload-time = "2025-10-16T22:16:39.375Z" }, + { url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890, upload-time = "2025-10-16T22:16:40.547Z" }, + { url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472, upload-time = "2025-10-16T22:16:41.694Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" }, + { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" }, + { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101, upload-time = "2025-10-16T22:16:49.318Z" }, + { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158, upload-time = "2025-10-16T22:16:50.517Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360, upload-time = "2025-10-16T22:16:52.646Z" }, + { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548, upload-time = "2025-10-16T22:16:57.008Z" }, + { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065, upload-time = "2025-10-16T22:16:58.206Z" }, + { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384, upload-time = "2025-10-16T22:16:59.36Z" }, + { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, +] + [[package]] name = "verspec" version = "0.1.0" @@ -4482,6 +6659,87 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/2a/dc2228b2888f51192c7dc766106cd475f1b768c10caaf9727659726f7391/virtualenv-20.36.1-py3-none-any.whl", hash = "sha256:575a8d6b124ef88f6f51d56d656132389f961062a9177016a50e4f507bbcc19f", size = 6008258, upload-time = "2026-01-09T18:20:59.425Z" }, ] +[[package]] +name = "vllm" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "anthropic" }, + { name = "apache-tvm-ffi" }, + { name = "blake3" }, + { name = "cachetools" }, + { name = "cbor2" }, + { name = "cloudpickle" }, + { name = "compressed-tensors" }, + { name = "depyf" }, + { name = "diskcache" }, + { name = "einops" }, + { name = "fastapi", extra = ["standard"] }, + { name = "fastsafetensors" }, + { name = "filelock" }, + { name = "flashinfer-cubin" }, + { name = "flashinfer-python" }, + { name = "gguf" }, + { name = "ijson" }, + { name = "lark" }, + { name = "llguidance", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 'x86_64'" }, + { name = "lm-format-enforcer" }, + { name = "mcp" }, + { name = "mistral-common", extra = ["image"] }, + { name = "model-hosting-container-standards" }, + { name = "msgspec" }, + { name = "ninja" }, + { name = "numba" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "nvidia-cudnn-frontend" }, + { name = "nvidia-cutlass-dsl" }, + { name = "openai" }, + { name = "openai-harmony" }, + { name = "opencv-python-headless" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp" }, + { name = "opentelemetry-sdk" }, + { name = "opentelemetry-semantic-conventions-ai" }, + { name = "outlines-core" }, + { name = "partial-json-parser" }, + { name = "pillow" }, + { name = "prometheus-client" }, + { name = "prometheus-fastapi-instrumentator" }, + { name = "protobuf" }, + { name = "psutil" }, + { name = "py-cpuinfo" }, + { name = "pybase64" }, + { name = "pydantic" }, + { name = "python-json-logger" }, + { name = "pyyaml" }, + { name = "pyzmq" }, + { name = "quack-kernels" }, + { name = "regex" }, + { name = "requests" }, + { name = "sentencepiece" }, + { name = "setproctitle" }, + { name = "setuptools", marker = "python_full_version >= '3.12'" }, + { name = "six", marker = "python_full_version >= '3.12'" }, + { name = "tiktoken" }, + { name = "tilelang" }, + { name = "tokenizers" }, + { name = "torch" }, + { name = "torchaudio" }, + { name = "torchvision" }, + { name = "tqdm" }, + { name = "transformers" }, + { name = "typing-extensions" }, + { name = "watchfiles" }, + { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/80/9798ce5e16af5754183ef33a63dc27017e2b51c87f51cc741832ce47a2d5/vllm-0.20.0.tar.gz", hash = "sha256:a6d50152936ee292455af3ffbe359f7a284ac43bf3b68caccf29f368e196cc72", size = 33508260, upload-time = "2026-04-27T11:08:04.666Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/5b/26379d3c522379373e50b9f77adf55eb94f4a0f62a6c8e3e7fe3f0bf0d39/vllm-0.20.0-cp38-abi3-manylinux_2_35_aarch64.whl", hash = "sha256:29a135ca0d70650f057f15c7c0b560d24659524c771f70fbddc24597c861c118", size = 235776358, upload-time = "2026-04-27T11:07:22.058Z" }, + { url = "https://files.pythonhosted.org/packages/47/bb/cb02d1e9679fce892a674f86caee25acc9ddd64d7dafa4cfe29e899993a8/vllm-0.20.0-cp38-abi3-manylinux_2_35_x86_64.whl", hash = "sha256:24d28892e210200f6e1bd13f699c42a74cd2bb7364c11248e2348f677c7f6dfb", size = 244415937, upload-time = "2026-04-27T11:07:48.135Z" }, +] + [[package]] name = "wandb" version = "0.27.2" @@ -4556,6 +6814,82 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, ] +[[package]] +name = "watchfiles" +version = "1.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/41/5e1a4bb12aac5f1493fa1bdc11154eca3b258ca4eba65d39c473fe19d8e9/watchfiles-1.2.0.tar.gz", hash = "sha256:c995fba777f1ea992f090f9236e9284cf7a5d1a0130dd5a3d82c598cacd76838", size = 108252, upload-time = "2026-05-18T04:32:04.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/d8/19483ef075d601c409bce8bcbb5c0f81a10876fff870400568f08ce484a1/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89d8c2394a065ca86f5d2910ff263ae67c127e1376ccc4f9fc35c71db879f80a", size = 456611, upload-time = "2026-05-18T04:30:45.723Z" }, + { url = "https://files.pythonhosted.org/packages/b1/6a/cc81fbe7ee42f2f22e661a6e12def7807e01b14b2f39e0ff83fd373fd307/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:772b80df316480d894a0e3165fdd19cf77f5d17f9a787f94029465ad0e3529d1", size = 461379, upload-time = "2026-05-18T04:31:29.292Z" }, + { url = "https://files.pythonhosted.org/packages/b1/57/7e669002082c0a0f4fb5113bb70125f7110124b846b0a11bc5ae8e90eac1/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d158cd89df6053823533e06fb1d73c549133bff5f0396170c0e53d9559340717", size = 493556, upload-time = "2026-05-18T04:30:05.44Z" }, + { url = "https://files.pythonhosted.org/packages/45/7d/f60a2b19807b21fe8281f3a8da4f59eef0d5f96825ac4680ba2d4f2ebf91/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d516b3283a758e087841aedb8031549fb41ced08f3db10aa6d2bf32dc042525b", size = 575255, upload-time = "2026-05-18T04:30:40.568Z" }, + { url = "https://files.pythonhosted.org/packages/bd/49/77f5b5e6efbcd57482f74948ebb1b97e5c0046d6b61475042d830c84b3ff/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:53b2290c92e0506d102cd448fbc610d87079553f86caa39d67440856a8b8bba5", size = 467052, upload-time = "2026-05-18T04:31:17.942Z" }, + { url = "https://files.pythonhosted.org/packages/ee/5a/73e2959af1b97fd5d556f9a8bdba017be23ceeef731869d5eaa0a753d5a3/watchfiles-1.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a711b51aec4370d0dcda5b6c09463206f133a5759341d7744b953a7b62e1100e", size = 456858, upload-time = "2026-05-18T04:30:30.182Z" }, + { url = "https://files.pythonhosted.org/packages/50/57/1bc8c27fad7e6c19bddee15d276dbb6ab72480ec01c127afff1673aee417/watchfiles-1.2.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:e2ca07fa7d89195ec0865d3d285666286740bfa83d83e5cee204043a31ecc165", size = 467579, upload-time = "2026-05-18T04:32:15.897Z" }, + { url = "https://files.pythonhosted.org/packages/09/6c/3c2e44edba3553c5e3c3b8c8a2a6dee6b9e12ae2cf4bd2378bebf9dc3038/watchfiles-1.2.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e0618518f282c4ebff60f5e5b1247b6d91bb8b9f4476947563a1e74acc66f3c6", size = 633253, upload-time = "2026-05-18T04:31:37.123Z" }, + { url = "https://files.pythonhosted.org/packages/30/c2/d8c84a882ab39bbefcc4915ab3e91830b7a7e990c5570b0b69075aba3faf/watchfiles-1.2.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0d191c054d0715c3c95c99df9b8dbf6fd096d8c1e021e8f212e1bd8bc444ccb5", size = 660713, upload-time = "2026-05-18T04:31:24.62Z" }, + { url = "https://files.pythonhosted.org/packages/13/01/b32a967c56fb3e3e5be3db52c3d3b87fa4513aa367d8ed1ad96d42952e5f/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f70d8b291ef6e88d19b1f297a6905ddb978888d9272b0d05e6f53309856bcfc", size = 453207, upload-time = "2026-05-18T04:31:04.231Z" }, + { url = "https://files.pythonhosted.org/packages/04/98/97557a812180338cb1abd32e1cffcc4588f59b5f23e0cb006b2ba95ba64a/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d8641cf834c2836922899105bd3ce3d0dfc69291d52edf0b4d0436829b34c0", size = 459273, upload-time = "2026-05-18T04:31:50.377Z" }, + { url = "https://files.pythonhosted.org/packages/e8/a8/b4b08dcb7653b8087c6586f7ce649505900e866bbcfe40dc9587af02e686/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2581a94056e55d7d0a31a823ea92bf73749c489ca2285bfdc0fbe6b2bb49d50c", size = 489927, upload-time = "2026-05-18T04:31:42.485Z" }, + { url = "https://files.pythonhosted.org/packages/50/94/3dceea03545d2e5ddfd839f0ddd5e1cecbf1697b5a428d5ba11cef6af95d/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:41bc1199f7523b3f82843c88cbb979180c949caef0342cf90968f178e5d49b01", size = 570476, upload-time = "2026-05-18T04:31:03.071Z" }, + { url = "https://files.pythonhosted.org/packages/cc/f2/d39a5450c3532092b91f81d274360e613c2371bc874a89c7a1a3c5e8d138/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7571e4464cb6e434958f867f7f730b8ab0b75e3f8e5eac0499168486ab3c33a8", size = 465650, upload-time = "2026-05-18T04:30:12.701Z" }, + { url = "https://files.pythonhosted.org/packages/22/24/ed72f68cbc1333ca9b9f2200aa048bb6658ae41709bc1caad4310f4bdffd/watchfiles-1.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e53a384f76b631c3ae5334ce6a52f0baa3a911eb94a4eac7f160079868b716d5", size = 456398, upload-time = "2026-05-18T04:30:13.784Z" }, + { url = "https://files.pythonhosted.org/packages/0d/64/982ef4a4e5bab5b6e5b6becc8cd5e732f6130a78b855f0abec6439a9a135/watchfiles-1.2.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d20029a60a71a052a24c4db7673bc4de39ab89adbaccbfb5d67987c5d73f424d", size = 465140, upload-time = "2026-05-18T04:31:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/a0/0c/95282abf4ed680b6096010bcfc30c5fa7a041fc5aa5a2ad17a2cc6c75bba/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:2cb93af48550faf1cea04c303107c8b75833de7013e57ce27d3b8d21d8d0f58c", size = 630259, upload-time = "2026-05-18T04:31:25.676Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/607c1de1530c4bdcf2cf1d1ecc2505ddba5d96bd43ba9f2b0e79876f850f/watchfiles-1.2.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2995c176de7692b86a2e4c58d9ec718f753150a979cb4a754e2b4ffa38e70906", size = 659859, upload-time = "2026-05-18T04:30:24.333Z" }, + { url = "https://files.pythonhosted.org/packages/db/ef/d5bdb705c224dbc256aa0c1ec47bf4e61ec52558f2afb44a71a1fe4d7015/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f34e26a19f91f710c08e0183429f0d1d15df734e6bc78c31e77b9ea9c433658", size = 452989, upload-time = "2026-05-18T04:31:11.945Z" }, + { url = "https://files.pythonhosted.org/packages/71/29/5495f2c1661949ef7a35e4d71111d129cfe7606414a26887a919d0a55406/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b4e77f6a55f858504069abd35d336a637555c09bca453dde1ee1e5ada8a6a1fb", size = 458978, upload-time = "2026-05-18T04:30:52.606Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8c/7f9c07c433811c2fffd93e13fdfb7135de9aab5f2ae41be08960fa0047dc/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0cb4d80e212f116474a545c21c912b445f16bb0cef9e6a73a498164223e14e2f", size = 490248, upload-time = "2026-05-18T04:31:36.003Z" }, + { url = "https://files.pythonhosted.org/packages/3c/11/d93632febc52fbc21be90231bb7c17fd5387f46c9076fd40a5f9c2ae6910/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b974946a10af379d425e2eef5b62f5c6ebeaccf91d45eaad6f5b27ecd4f91aa0", size = 571847, upload-time = "2026-05-18T04:31:10.862Z" }, + { url = "https://files.pythonhosted.org/packages/55/b4/383173e73aabb07ad1d9c7aa859d95437ac46a6d6a1e11005facda0c9d19/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86bc13c25a8d1fcd70b51d0ce7c9b65e90de5666fcbfd3e34957cc73ee19aeb5", size = 465974, upload-time = "2026-05-18T04:30:17.006Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6c/89b1a230a78f57c52dd8893adb1f92f94411721b6ec12596c56d98c74356/watchfiles-1.2.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca148d73dea36c9763aaa351e4d7a51780ec1584217c45276f4fe8239c768b71", size = 454782, upload-time = "2026-05-18T04:30:35.656Z" }, + { url = "https://files.pythonhosted.org/packages/24/62/1732118367cfff0a9fce3bf62ff4bfded09ef5df21d9d446b858b3f70a96/watchfiles-1.2.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:c525543d91961c6955b2636b308569e84a1d1c5f5f2932041ab9ef46422f43e3", size = 465182, upload-time = "2026-05-18T04:30:20.846Z" }, + { url = "https://files.pythonhosted.org/packages/28/96/716f7e5f51339bf22963f3345f9f27d7f3b30e2eadc597e257c881dd3c53/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:a204794696ffb8f9b10fba6f7cb5216d42f3b2b71860ccac6b6e42f5f10973b0", size = 629841, upload-time = "2026-05-18T04:31:05.397Z" }, + { url = "https://files.pythonhosted.org/packages/4c/fe/c40783950fd771ccf66ab3ec2722d188a9af1c7f96c6e811f36e40c6e03f/watchfiles-1.2.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:10d86db20695afe7997ac9e1717637d6714a8d0220458c33f3d2061f54cec427", size = 658028, upload-time = "2026-05-18T04:31:48.22Z" }, + { url = "https://files.pythonhosted.org/packages/84/2c/0a44fe058cb4bb7b8ede6b6670698bbb7c0400740e378d00022189b7b31d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fff610d7bb2256a317bb1e96f0d7862c7aa8076733ee5df0fd41bbe76a24a4f4", size = 451892, upload-time = "2026-05-18T04:32:14.005Z" }, + { url = "https://files.pythonhosted.org/packages/67/a1/351e0d56cd35e6488b5c8b4fb11a809a5bc923e8fe8fed9faf8920be0c89/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b141a4891c995a039cd89e9a49e62df1dc8a559a5d1a6e4c7106d16c12777a55", size = 458867, upload-time = "2026-05-18T04:31:22.279Z" }, + { url = "https://files.pythonhosted.org/packages/d5/7d/9d09605187f1b838998624049fcf8bf47b73c1a3b76901fcac1782f62277/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f22943b7770483f6ea0721c6b11d022947a98eb0acae14694de034f4d0d38925", size = 490217, upload-time = "2026-05-18T04:31:43.657Z" }, + { url = "https://files.pythonhosted.org/packages/60/5d/a17a16eccb182f04188cd308ec24b1a71a9b5c4e7098269cf35d9fa56d02/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1bc6195825b7dcd217968bb1f801a60fd4c16e8eeab5bedc7fe917d7d5995ab4", size = 571458, upload-time = "2026-05-18T04:32:11.875Z" }, + { url = "https://files.pythonhosted.org/packages/d3/3d/4dd457062083ab1938e5dfd45032eb425cee2ac817287ca8ff4356183e5d/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4a4b147f5dca2a5d325a06a832fb43f345751adfbc63204aec30e0d9ca965a2", size = 464707, upload-time = "2026-05-18T04:30:43.492Z" }, + { url = "https://files.pythonhosted.org/packages/c6/71/ea8c57b128f5383de74d0c7d2d9c57ad7c9a65a930c451bd25d524b295b7/watchfiles-1.2.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4543579a9bdb0c9560039b4ffddbdb39545707659fbc430ce4c10f3f68d557f9", size = 454663, upload-time = "2026-05-18T04:30:16.061Z" }, + { url = "https://files.pythonhosted.org/packages/53/fd/2e812bf938406d7db351f0703ddd3fc6c061cf30d96153a77bc79a943a44/watchfiles-1.2.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:20aa0e708b920bde876a4aa82dc7dd6ebea228a63a67cda6632c2fc87b787efa", size = 463537, upload-time = "2026-05-18T04:31:44.9Z" }, + { url = "https://files.pythonhosted.org/packages/86/56/d17a7f1dd1bc3035f1072694a551301272f1739c2d8e319c927cb9e29b38/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:d413349d565dab74297f2a63e84a097936be69bf8f3b3801f27f380e32040f44", size = 629194, upload-time = "2026-05-18T04:31:14.141Z" }, + { url = "https://files.pythonhosted.org/packages/be/06/f1ff66bf5cae50aa4062779a0ecd0bbaf15e466195719074078947d9a17d/watchfiles-1.2.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:f28b2725eb8cce327b9b3ab02415c853011dc55c95832fe90de6bc56f5315f72", size = 656194, upload-time = "2026-05-18T04:31:47.14Z" }, + { url = "https://files.pythonhosted.org/packages/26/ca/1ad30103535cf0cecd7b993e8d50edc5351b1820e38f2d22e3df58962feb/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7ce236284f002a156f70add88efe5c70879cccbb658be0822c54b1306fc09d", size = 452448, upload-time = "2026-05-18T04:30:53.727Z" }, + { url = "https://files.pythonhosted.org/packages/37/a1/ceee2cdf2afbd715fa07758d39c9859513eae411b23196f7fd039e5feedd/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b9909cc2b48468b575eefa944919e1fe8a36c5849d5c7c168f80a8c1db69398e", size = 459605, upload-time = "2026-05-18T04:30:23.312Z" }, + { url = "https://files.pythonhosted.org/packages/e8/f6/421e30fd1cb3907a84ed92ab3f1983e37ba2dca015e9a894a048418417a2/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a37faaed405c67e28e6be45a1fa4f206ef5a2860f27c237db9fa30704c38242", size = 490757, upload-time = "2026-05-18T04:30:47.358Z" }, + { url = "https://files.pythonhosted.org/packages/41/b0/55ed1b97ed08be7bba6f9a541cac15f2a858e1d74d2b07b6da70a82aab00/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9649193aa27bd9ff2e80ff29bfaa93085496c7a3a377592823cc58b77ee88add", size = 568672, upload-time = "2026-05-18T04:30:38.915Z" }, + { url = "https://files.pythonhosted.org/packages/d1/cf/d8ae8a80dd7bafab395ea7681c10237311bbf34d37704a8c744e7cf31fc7/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e4ff8e37f99cf1da89e255e07c9c4b37c214038c4283707bdec308cb1b0ea1f", size = 464197, upload-time = "2026-05-18T04:30:09.914Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/3076c496ca8dafe0e8cd03fcebdfc47be4b1174b4e5b24ff6e396e6b3af2/watchfiles-1.2.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:054dc20fd2e3132b4c3883b4a00d72fd6e1f56fdaf89fccd12e8057d74cd74d7", size = 453181, upload-time = "2026-05-18T04:30:14.829Z" }, + { url = "https://files.pythonhosted.org/packages/e5/10/9745e17c98e7b8a86454df0a3c7b5686bd650383f1e9f26e4ebcbd6cc0c0/watchfiles-1.2.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e140ed30ebde76796b686e67c182cff10ea2fbab186fafd1560f74bb5a473a6e", size = 465109, upload-time = "2026-05-18T04:30:28.123Z" }, + { url = "https://files.pythonhosted.org/packages/8f/95/8ef4a95481d3e0cb52d62a06fa6e972e81424be2d9698b91a2fecca9904c/watchfiles-1.2.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:bb7e52ecf68ba46d22df23467b87cffeb2146908aa523ebfe803019618cfda06", size = 630653, upload-time = "2026-05-18T04:31:49.304Z" }, + { url = "https://files.pythonhosted.org/packages/fd/e4/3b3bf36b0f829b50c6ebcb8d031583863c59f923d6a6af3d485e470d0fac/watchfiles-1.2.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:23282a321c8baf9b3a3c4afff673f9fe65eb7fdc2338d765ccad9d3d1916a5ba", size = 657838, upload-time = "2026-05-18T04:31:06.497Z" }, + { url = "https://files.pythonhosted.org/packages/47/c8/bec626bcc2d69f44b9acb24ce7d60ed7b16b73628eea747fcbd169d8edda/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4429f3b105524a10b72c3a819b091c495d2811d419c1e1e8df773a5a5974f831", size = 452409, upload-time = "2026-05-18T04:31:20.142Z" }, + { url = "https://files.pythonhosted.org/packages/00/b7/b6362068e81e7c556d155a34c35d40ac3ef42d747b06d7f6e5bf58e359c2/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:43d818978d06062d9b22c4fab2ebe44cf5213d42dc8e62bda8c2760cfa2eeb33", size = 458827, upload-time = "2026-05-18T04:32:06.219Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/9a813fa42afb1e0b4625e75f0479826644d3ee8dc287e093799bc01f390c/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b9f732dc58b2dbe69e464ccf8fff7a03b0dd0be439da4c0720d3558527d3d6b4", size = 490104, upload-time = "2026-05-18T04:31:56.034Z" }, + { url = "https://files.pythonhosted.org/packages/2f/bf/27dfb6094ca4c9aad21298b5525b6c53cb36121ee454331d05161e58d130/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8f200104103feb097de4cab8fe4f5dd18a2026934c7dea98c55a2f5fd6d5a33b", size = 571360, upload-time = "2026-05-18T04:31:57.133Z" }, + { url = "https://files.pythonhosted.org/packages/fb/39/44a096d67270ea93df91d33877dbe91fbda3aa4f8ec2edf799d93eda8736/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:63ac26eefbf4af1741247d6fb68b11c49a25b2f7413fbd318a83a12aaa9cf666", size = 464644, upload-time = "2026-05-18T04:30:57.33Z" }, + { url = "https://files.pythonhosted.org/packages/0e/80/c7472203bad6268e3ef1ad260739704847898938ad7ea8b63a5131f46b50/watchfiles-1.2.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c4997d4e4a55f0d02b6cde327322daf3a0400e5df6c6b15948994bf72497925", size = 454771, upload-time = "2026-05-18T04:30:48.736Z" }, + { url = "https://files.pythonhosted.org/packages/51/cf/3b10b268b4b7f0fc26e9debb5eef1998b515887840f444cd3ec80c688755/watchfiles-1.2.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:4c887eba18b7945ac73067a8b4a66f21cd46c2539b2bc68588f7be6c7eb6d26b", size = 463494, upload-time = "2026-05-18T04:31:33.826Z" }, + { url = "https://files.pythonhosted.org/packages/3d/3e/a4302545cd589262a0dc7d140e86f7688eba3f9c72776c27f7e23b8864c4/watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:3416ff151bb6b5a8d8d11664974fbef4d9305b9b2957839ab5a270468fd8df30", size = 629383, upload-time = "2026-05-18T04:31:15.596Z" }, + { url = "https://files.pythonhosted.org/packages/db/99/d5649df0a9a410d45b7c882304d0b790903ac9b6e8f2cfd12114e0c6b9f2/watchfiles-1.2.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:0e831a271c035d89789cffc386b6aa1375f39f1cd25eb7ca0997e4970d152fc5", size = 656093, upload-time = "2026-05-18T04:31:58.707Z" }, + { url = "https://files.pythonhosted.org/packages/3c/01/c66dd95d0423fe30d31820e2d1d5bda773764131bbb6ac0cb1cf303ac328/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a105bc2283f67e8fbec74253ec2d94925de92ed72c0393f1206bf326b7b7b69", size = 452372, upload-time = "2026-05-18T04:31:00.836Z" }, + { url = "https://files.pythonhosted.org/packages/91/15/2fe99557e72f85627c6a8eed50d889e8d101623e060a22ad75b875cb932d/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5327989a465505f05cfe06f04fa9d0c2fd5432bb243e10e6f012b1bdca3c8579", size = 459596, upload-time = "2026-05-18T04:31:34.96Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/d4acfa0023367428ed48351b3b9b267893037b6cadae55620c61c24bcfd4/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ecb47f183a8025b2aa18b546725c3657e542112ae9c0613a2af79b4fa8d04ad7", size = 490869, upload-time = "2026-05-18T04:31:59.923Z" }, + { url = "https://files.pythonhosted.org/packages/a4/5f/3164cbdce06c9fb95c4f7b9e2f9760b5e2797af43a9ecc317ef42a23a278/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8520a4ab0e37f770afc34459c4f8f7019e153f9124dc101c15538365875d1ab2", size = 571641, upload-time = "2026-05-18T04:32:00.948Z" }, + { url = "https://files.pythonhosted.org/packages/41/e6/85d3731c55e65cd7690f3f803d24c139588aaf863e4bf2148fe7a7fa1a19/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:71cd71740ed2c15211ebb237ced4e39a1cdf6f80566e5fe95428da1626f4fde6", size = 464444, upload-time = "2026-05-18T04:30:34.298Z" }, + { url = "https://files.pythonhosted.org/packages/f4/7d/562641012b8b09872742c3b8adf9629ec479fd78f8d68ae4a0c13da8add6/watchfiles-1.2.0-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f88af53d6ddaf72179ef613ddc905e6f4785f712b49b80b3bef9f3525e6194b4", size = 453593, upload-time = "2026-05-18T04:31:23.464Z" }, + { url = "https://files.pythonhosted.org/packages/56/fe/cb8ef3d6f929d14158fdaaad9925985b7310abc9384dcd4d82dd0016fb59/watchfiles-1.2.0-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:cee9d5efd929efdac5f7e58f72b3376f676b64050a91c5b99a7094c5b2317488", size = 465096, upload-time = "2026-05-18T04:31:30.384Z" }, + { url = "https://files.pythonhosted.org/packages/25/91/80908e835e100527a9267147b08c0eee1fa6ab0ffec15edc04d1d44885f7/watchfiles-1.2.0-cp315-cp315-musllinux_1_1_aarch64.whl", hash = "sha256:b718bf356bbc15e559bd8ef41782b573b8ae0e3f177ab244b440568d7ea02cfb", size = 630638, upload-time = "2026-05-18T04:30:49.89Z" }, + { url = "https://files.pythonhosted.org/packages/46/4b/95ab2f256bb4af3cb2eb23b9317bda984ee6e0f11733a5c004a6c95b06e3/watchfiles-1.2.0-cp315-cp315-musllinux_1_1_x86_64.whl", hash = "sha256:922c0e019fe68b3ae392965a766b02a71ba1168c932cebc3733cd52c5fe5b377", size = 657684, upload-time = "2026-05-18T04:31:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2c/73f31a3b893886206c3f54d73e8ad8dee58cdb2f69ad2622e0a8a9e07f4e/watchfiles-1.2.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0ef001f8c25ad0fa9529f914c1600647ecd0f542d11c19b7894768c67b6acb7", size = 457318, upload-time = "2026-05-18T04:31:01.932Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f9/45d021e4a5cc7b9dd567f7cbb06d3b75f751a690063fb6cc7ec60f4e46b7/watchfiles-1.2.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a88fc94e647bc4eec523f1caa540258eb71d14278b9daf72fa1e2658a98df0f0", size = 457771, upload-time = "2026-05-18T04:30:56.331Z" }, +] + [[package]] name = "wcwidth" version = "0.6.0" @@ -4592,6 +6926,88 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/34/db/b10e48aa8fff7407e67470363eac595018441cf32d5e1001567a7aeba5d2/websocket_client-1.9.0-py3-none-any.whl", hash = "sha256:af248a825037ef591efbf6ed20cc5faa03d3b47b9e5a2230a529eeee1c1fc3ef", size = 82616, upload-time = "2025-10-07T21:16:34.951Z" }, ] +[[package]] +name = "websockets" +version = "17.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f7/96/e01084f83a64bcb3a27994bd0cb0db68ff29d9c6707fae37ec19b18ba990/websockets-17.0.1.tar.gz", hash = "sha256:5baa9bc0dfbae8c507e51c8cf1b6d4628086f7a87bbd3a9952bd5f035451f1cc", size = 183298, upload-time = "2026-07-31T11:31:27.665Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/ab/13a856b488dbac7fd3c5473e38098897cda0baa04e3ca3b34ec6c8a32b46/websockets-17.0.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b98860aefbd3d9bc8e3c7f0eefb83b11142b16110739c68cd33d3b4d6e84e536", size = 219602, upload-time = "2026-07-31T11:29:09.227Z" }, + { url = "https://files.pythonhosted.org/packages/45/a1/2b100e71aa1fe283ec8fb73f8b74a8576d855486a49117903b100dd3b78d/websockets-17.0.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d41e9845514754a42d1d83b2fca9d27fee2ca7b3b0bee6843ba5a9bb2b6e25ac", size = 219873, upload-time = "2026-07-31T11:29:10.362Z" }, + { url = "https://files.pythonhosted.org/packages/3b/71/92e6146d3588d145136c0e0e16d106bbb855bb5047e13ec3fdee39cce770/websockets-17.0.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9aac6081513f02eac3f8caace800dbfc5c608b69e4a7bef69e414eabfc95aa1", size = 221108, upload-time = "2026-07-31T11:29:11.708Z" }, + { url = "https://files.pythonhosted.org/packages/09/8d/357afa2ffd29686536109e7e3cb2a94f2d09e535df4cf3989393dc506a40/websockets-17.0.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:b85b960a4507b0714c0a1246d031be9118d908ee974dc085257297a955205f1d", size = 224401, upload-time = "2026-07-31T11:29:12.959Z" }, + { url = "https://files.pythonhosted.org/packages/01/27/9efba1e7a8df48e405d017e67513c6b2a8f0b59c8500b820c47cd5f3dea9/websockets-17.0.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c356dbddab0a529ed7574f78f559d75a223735c321c28f6f587fbf02b11ed301", size = 221670, upload-time = "2026-07-31T11:29:14.199Z" }, + { url = "https://files.pythonhosted.org/packages/01/85/ab27d62103e8a150f3657e043e5fc711ad3e018c8cd8a715093e93d7640c/websockets-17.0.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5661f868ef191d33dfc6a0cc7c5b3d495f0cc8bb3f8b30d87bda8755c61c95f5", size = 220442, upload-time = "2026-07-31T11:29:15.417Z" }, + { url = "https://files.pythonhosted.org/packages/6d/04/289e00b8001b622b0c397a6901fcc4aa8f34a6d2ed42be17f2704f2faae1/websockets-17.0.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2fa2cb465a131c347ba6717a78c887746e73edb1c131d01c982d6ef0d68b82e0", size = 217760, upload-time = "2026-07-31T11:29:16.772Z" }, + { url = "https://files.pythonhosted.org/packages/2f/70/0fe58cdac988dfc0066786cc07b09dfd72b48b0c01e5de667721192b2e6e/websockets-17.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:55383d8177b3c99fd873ee5db0e0193f4c1dd4a3feaccf1a4a03c1b7cf539cac", size = 220597, upload-time = "2026-07-31T11:29:18.075Z" }, + { url = "https://files.pythonhosted.org/packages/d5/76/d3eaf120710d1a791d1c7a4963f0b50a589df8ba675394fd2a97dfea3746/websockets-17.0.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:038cfad5d5417f8bb09295abe986029a26d22f34bda622ccc79b670efd4dab56", size = 219187, upload-time = "2026-07-31T11:29:19.468Z" }, + { url = "https://files.pythonhosted.org/packages/e6/00/4e9ae886bdb1647537176ca33fca66d79dddb3773d213ac98ddc6bba9ab4/websockets-17.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:15920057a6b723f84734f0641403bca163a4b176e5af809ee4f0c4a1e75e9fed", size = 219955, upload-time = "2026-07-31T11:29:20.641Z" }, + { url = "https://files.pythonhosted.org/packages/7c/67/cd7cc6849a86cf8c9979c0c570b6b6ccee75d2872854238d8d54e77be6ec/websockets-17.0.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:5508f38c98ac29def9e747b87543b008a58b075df6da70b2cf2e0b47073d33bb", size = 221002, upload-time = "2026-07-31T11:29:21.753Z" }, + { url = "https://files.pythonhosted.org/packages/de/5c/4d14eaf7b2f1448d1af24c1641f04eb74c1632a5802952aac4b7e068b8e7/websockets-17.0.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:a68e604c6d1b0338e46652e2688cbce8096ad9c03548b075fda9e2ea19a9b7dd", size = 218579, upload-time = "2026-07-31T11:29:22.888Z" }, + { url = "https://files.pythonhosted.org/packages/15/67/ff8bc4b8a6ec235ed8985de12fecc59ad2cd68cc8fc79b97deaa42e412ac/websockets-17.0.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:a8af570fc29cd998a921c7131c8ac81d9434466d6d25300cb12a690fb56a8a08", size = 219612, upload-time = "2026-07-31T11:29:24.052Z" }, + { url = "https://files.pythonhosted.org/packages/91/eb/103d81d655bab3ffd5c7d5d4b08f92c374499decd1d4be6035ce715b385b/websockets-17.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:246927ae9ae06ca0d42a483a4bdb80d4862e1ee5b4cab37c354a5e1ad8356448", size = 219846, upload-time = "2026-07-31T11:29:25.421Z" }, + { url = "https://files.pythonhosted.org/packages/cd/2f/2940e57080cf56f28190287516400126d5a76b52b9a61dc10ba6f6400dbe/websockets-17.0.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:c09e097d0e46e3c289bedab9a475ae344b70c30ff5646e46af22b4e6fdc97b21", size = 219874, upload-time = "2026-07-31T11:29:34.608Z" }, + { url = "https://files.pythonhosted.org/packages/42/28/9ec976c16d63cc51c28dfec74b66854048c0b8b6579946e902f91b69e8bf/websockets-17.0.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:f47b0815af3948ec6a440b3afa02f05b18cc0939549e91b5c677b5d9c2c8472a", size = 220150, upload-time = "2026-07-31T11:29:35.831Z" }, + { url = "https://files.pythonhosted.org/packages/f4/a4/850c699a16bbc451723856360c59bd997bec075e637154f3fa96e80d5760/websockets-17.0.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8848c207049ad49d318e5f64a3d4d7bb189f8328d0d98e65647788f2a085785c", size = 221389, upload-time = "2026-07-31T11:29:37.189Z" }, + { url = "https://files.pythonhosted.org/packages/47/e1/f60a891c1a4b3420d5052333a84eb7241e1fb4a71866dec1562f5fa30027/websockets-17.0.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2604de7228506b13a44a256a9d223943340c0e725af5d367dc068e192b027761", size = 224169, upload-time = "2026-07-31T11:29:38.61Z" }, + { url = "https://files.pythonhosted.org/packages/26/fb/e2a893be6fae4fddfe50ddc3035a331d3f381103d5467b7900026bdb3a64/websockets-17.0.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:07abc3bd196a48af476a82fd47f3f79a6a3f70937a9f930cef703cfa0c9d83b6", size = 222025, upload-time = "2026-07-31T11:29:39.897Z" }, + { url = "https://files.pythonhosted.org/packages/d9/72/e3144b2d79276fab9798ed7d4aea2f0847434f186800b6f56a1eddcb3114/websockets-17.0.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:769ce7e2acfd9a89f2bed3a9c0da229459516bbc00bd4c9e2ca492c613ae4861", size = 220779, upload-time = "2026-07-31T11:29:41.084Z" }, + { url = "https://files.pythonhosted.org/packages/c0/5e/69c02174fbcf1c40c6adc45d3c316a401558392fe7bab8969ef8c46f1689/websockets-17.0.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:07d78a509c3333f5908c83d7f78144ea68a6c9ec28110f5c54d81d8fcdc262c4", size = 218053, upload-time = "2026-07-31T11:29:42.322Z" }, + { url = "https://files.pythonhosted.org/packages/b3/09/7574778b095b99cfa0856583462f56568df784f9b41485145169b2ec9c64/websockets-17.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ffad64ce7ad3703d652a3fd9af26238377d24ce52c6ad8ff35d26d82f61f493f", size = 220825, upload-time = "2026-07-31T11:29:43.553Z" }, + { url = "https://files.pythonhosted.org/packages/c5/e6/f46571f38765dbc4cbc0d0b47de8db65768006dbbd4340e6f5f51bc1d895/websockets-17.0.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:e95e321d0d763f2b6633512605f6112ebd70d5746f3ce05c941909d4a25233f2", size = 219427, upload-time = "2026-07-31T11:29:44.731Z" }, + { url = "https://files.pythonhosted.org/packages/9d/31/6ff1fee057bd7e9dd5237fc064a749615378d003aa045b5bfc2d12b2f4f7/websockets-17.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:cd526c8228e759c1006c4b7c9ac71dc4e925ced1a6a6a5a8e94643709738f63e", size = 220198, upload-time = "2026-07-31T11:29:45.997Z" }, + { url = "https://files.pythonhosted.org/packages/7a/4f/d41847227a44b9ad87c3d5a9fddbfad8b7c4d6032878d8460d9d37c2d44f/websockets-17.0.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8cd3369e42c0246afaf9d669cfc19797e3a49e8c0a639544459c57597108b966", size = 221304, upload-time = "2026-07-31T11:29:47.314Z" }, + { url = "https://files.pythonhosted.org/packages/63/30/21a7e326c6ad2eb526cd5b816383d59cdeb28b8805b65a543c3cfbd8e8ce/websockets-17.0.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:b580794e926cab7ff42ee4371ef14e0b22cb2bb722a607f77769136468f49a3f", size = 218858, upload-time = "2026-07-31T11:29:48.587Z" }, + { url = "https://files.pythonhosted.org/packages/2d/48/55b0331cd5bec9ce29748f79edc00075805450a47011d0c8e3b1c61dbf04/websockets-17.0.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5033ffe6804dd53afafa7d08e8c3eef2d2431f34d58ca30507a8442dd04a033a", size = 219840, upload-time = "2026-07-31T11:29:49.791Z" }, + { url = "https://files.pythonhosted.org/packages/78/6e/2e8bc06e546f49b32a58a2bc2957902d1809ecc37552d3d7ccd6639a126e/websockets-17.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6db9e5bf3649ab506c6ae8a3ac85a00fb1ae3816d75962771b2df8adbc5d40d2", size = 220116, upload-time = "2026-07-31T11:29:51.026Z" }, + { url = "https://files.pythonhosted.org/packages/87/32/fbf2d132f63ba3e67f675bccf333469786a24e0418969ce1d8e6ff9e6f02/websockets-17.0.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:9493314a99e599163c854fb5900ad7f7ea38c5cb9d9103aa30b3c6b8181c01fa", size = 219925, upload-time = "2026-07-31T11:30:00.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/50/64eee3d25a47fe744a9490e0627cc373dca096755db740f91c28bd61cd35/websockets-17.0.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:18ded646ce98cdd3c0235825b3252f1df55765ba49b616bb10282f758667b4d0", size = 220206, upload-time = "2026-07-31T11:30:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/15/56/10ed4bc4dd75f204e3c62bd4898e44a8742a27773c80b188cfa7888aad2d/websockets-17.0.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c1bec5d6a19f5fbe87e4940739cfc65e7bb53d8b353e1029b8037a1653b321bc", size = 221445, upload-time = "2026-07-31T11:30:02.788Z" }, + { url = "https://files.pythonhosted.org/packages/bf/be/bb14328614c068ab09569962fbf218fc00413ce3febc6d2684c764b6f37e/websockets-17.0.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:872273e629ca7e3d35f16a2dc6ede84e1d5c831e616b8277de6e4f83114e7c58", size = 222887, upload-time = "2026-07-31T11:30:03.943Z" }, + { url = "https://files.pythonhosted.org/packages/32/1b/4cb0eec2fee310007104687493175af190019f705940c864f9c523fe9f6f/websockets-17.0.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1df81d174c1561292de9e40b141cafc04f69077272f6c352afe1d743e20810df", size = 222072, upload-time = "2026-07-31T11:30:05.258Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9c/14e6391de777ddb39c439c450deb551406d445e25a5877d6fa25c49d4544/websockets-17.0.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:759adeb5b0c5775b563254ec63b5b79089fc0045b479143a0b1b8c0ebaae1253", size = 220826, upload-time = "2026-07-31T11:30:06.53Z" }, + { url = "https://files.pythonhosted.org/packages/cb/57/96e94e384442247bbed5d3ab67381c7257355c2d66b62c3ad33a17f5d385/websockets-17.0.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4d1d99db29b5444e3982f1ce2ba8a833508ad44b2f1fbd0bd99e81d825c0b461", size = 218107, upload-time = "2026-07-31T11:30:07.766Z" }, + { url = "https://files.pythonhosted.org/packages/c0/8c/9c9dedd14c3919435df9b35cdee7111268c751252b87652f3a6a4f56e760/websockets-17.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:02ed63bf26dda9fa27df730a41f6664586c4ee05972c8fb667ce1725b3fd13d3", size = 220889, upload-time = "2026-07-31T11:30:09.035Z" }, + { url = "https://files.pythonhosted.org/packages/94/4d/ca73c2ac82c00f50c529784bacb323e42da4816333211bc1543d90c9cf11/websockets-17.0.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:eab6de8a98b9a7772cf686d00b4de439fc7efb8ab05ae106ef227291d06f87c5", size = 219486, upload-time = "2026-07-31T11:30:10.289Z" }, + { url = "https://files.pythonhosted.org/packages/5b/da/fb37ac09dcd7c69dd73bac979ed393df35f78a3c232e293d1ff3bd586d24/websockets-17.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2a855b6dfe21c4d3420be265ae031829ba8ba0be0ea350d9f7c3ef30ae63ebe2", size = 220258, upload-time = "2026-07-31T11:30:11.605Z" }, + { url = "https://files.pythonhosted.org/packages/fc/04/9693f191d968a93f37326a17301a101d49580889c688f466699f89ecdee1/websockets-17.0.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:7002d5f9e1c3ddd991cdfdbfee18cc8c8b196b2445022892badacd6cb338bbbc", size = 221358, upload-time = "2026-07-31T11:30:12.858Z" }, + { url = "https://files.pythonhosted.org/packages/cf/29/ad0d85c01db5dcf22898d51648bd2c25af0dd0a4a41c550b11acddeeeba7/websockets-17.0.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c395bda8e7d8f51a02e80261fb57127979e5c472675d9a96b2860619ad47da48", size = 218921, upload-time = "2026-07-31T11:30:14.064Z" }, + { url = "https://files.pythonhosted.org/packages/18/3b/bf8e855e495dcca63f2b8aa019cf2ada3160e1fa66d833c7417f3b1f7f38/websockets-17.0.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:aadc298969ad229d8e3029fc5cc751fdad286696230f9cf014e90ff9cd8e6ea0", size = 219871, upload-time = "2026-07-31T11:30:15.358Z" }, + { url = "https://files.pythonhosted.org/packages/3b/db/c7abd6639a93a40279cd1ddc57e09e1c4f8381c4cfccdb775aa5aac9770a/websockets-17.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f11a398d8170b7ac5000baf7f258dcda579ef3ea744e0cc6a165e0dfbc0d3198", size = 220154, upload-time = "2026-07-31T11:30:16.96Z" }, + { url = "https://files.pythonhosted.org/packages/cf/e3/88dc159d2ae66743c669443246243f28d873b0c5e58271b8cc1ca0440334/websockets-17.0.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:b3ff0ad440ad52dda64138f16895f66403f40192365e39b1010e889f289746b0", size = 219928, upload-time = "2026-07-31T11:30:26.221Z" }, + { url = "https://files.pythonhosted.org/packages/fe/f2/ff27eaefa15851a5cf7f004ab827a022bf2d6632cb520f89cb100db7e84b/websockets-17.0.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:72d7f2a5aeb4e82daa4ee18f125b4277f427033359be5c745ad709608446cc2c", size = 220279, upload-time = "2026-07-31T11:30:27.49Z" }, + { url = "https://files.pythonhosted.org/packages/19/2e/a5166149f363d2449c1cb2dde6486a245521979509d53b87a09f3e79662b/websockets-17.0.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6fd88365da261c53d3e943fb37e0d0721b9cde119f6b2e3fc84369b6ab234d63", size = 221525, upload-time = "2026-07-31T11:30:28.872Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/f46931269b3ff3bde65d27c65ddb22f9bb8ce92ac2c6c4df0910128f6219/websockets-17.0.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:ab9f962a5b64a5c3c845d556b7dc4e6fb683f7b67179f8205e814bb2e0213ffe", size = 222897, upload-time = "2026-07-31T11:30:30.164Z" }, + { url = "https://files.pythonhosted.org/packages/42/f4/deccf3439f35df953ec35e13fe07986821c5f1ab5785d69614283bdb9034/websockets-17.0.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8c07f145d0b9e90cbd96035f31fb79199aef4da1872854e36ebeb258e3d57594", size = 222129, upload-time = "2026-07-31T11:30:31.489Z" }, + { url = "https://files.pythonhosted.org/packages/35/a5/e1b57a59da92ade37fd021567a17b518ea8267b28e5530075844cdb525fe/websockets-17.0.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9f7747d3daa41a11f25f7cca5dc988fc51da97b311bed4c9d843860f79779283", size = 220875, upload-time = "2026-07-31T11:30:32.805Z" }, + { url = "https://files.pythonhosted.org/packages/f0/30/e7d0889c790a854156de424575fd67af79ddbaed9ff3157ae863dfd1c1dc/websockets-17.0.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:2abb1ba0a5133b7d2ef3c1c9f4b0c1e8a101012dce0b594ab2b2888d9a64820e", size = 218160, upload-time = "2026-07-31T11:30:34.512Z" }, + { url = "https://files.pythonhosted.org/packages/09/2e/43db785d6ed9ae7594fae7b62bbc9cb4dfee2b015e06a1005f1e5ce283b6/websockets-17.0.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:f3fd9a1f87f8f0f3f8e9f9bd0195f7516562d13f5b178db8c5784d1f60b60bed", size = 220951, upload-time = "2026-07-31T11:30:35.806Z" }, + { url = "https://files.pythonhosted.org/packages/eb/f5/4ac3cab3d5e8a830657a822f64a8910e3803229c6783e59c3fd9a3487427/websockets-17.0.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2bc14b481e05e331811108daa1aeb41a5e237a5564ef2f02ec5a356a0f102f78", size = 219460, upload-time = "2026-07-31T11:30:37.273Z" }, + { url = "https://files.pythonhosted.org/packages/03/0e/c3a4020673ffc17c82cf1a467835038a196a555d3b4f2a50f0f063cf8ccc/websockets-17.0.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:57d2ee9b24b404ce75f3814f92073c0ed88106c950148d2427fe8d25ca254d1f", size = 220248, upload-time = "2026-07-31T11:30:38.527Z" }, + { url = "https://files.pythonhosted.org/packages/7d/87/e47a6a278cc1dfade38444c893ce18322943c25d4b780a74450d9d164be1/websockets-17.0.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1b363bfd72a52c0658a3154a4cff219f15a474b35a235057d38853bf151acce7", size = 221421, upload-time = "2026-07-31T11:30:39.879Z" }, + { url = "https://files.pythonhosted.org/packages/da/8f/473d5fc4e3836e375b0233c6ef26777e6e5e3f7bfc84ccd524eae4090ed5/websockets-17.0.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:10b1587c599fa0f2c89154587c80e0fda98ade6c9fa8c0260a2823fb1800b685", size = 218975, upload-time = "2026-07-31T11:30:41.192Z" }, + { url = "https://files.pythonhosted.org/packages/d4/b9/819ec2dcdf69031d7e9cab11247f3a6ff9bbc8c7c53ada1dbcb9055b227b/websockets-17.0.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d7d72843691f50b91127c50688df10cb72ec6f4c4b1d7e2c11ab33b16acf8e51", size = 219925, upload-time = "2026-07-31T11:30:42.524Z" }, + { url = "https://files.pythonhosted.org/packages/85/b9/6c0da301f6118502e079cf92f4e864adf28e56b3f8c0f6085076ced7b876/websockets-17.0.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:90973a3a00f23afdfd1c9b06fb84289bf0220f247ef8a62501a1967c7af54f7b", size = 220218, upload-time = "2026-07-31T11:30:44.04Z" }, + { url = "https://files.pythonhosted.org/packages/48/18/7a77a82ce9d6f831c07b176da3942f7e71acd0f115f3ecdb1d00a040eb01/websockets-17.0.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:2503c7e2a5049a12d5dac917a46d5d52591283a766165b8176bb167560421b38", size = 220290, upload-time = "2026-07-31T11:30:53.582Z" }, + { url = "https://files.pythonhosted.org/packages/f5/af/43c3e3c3ea7ba4693c2181743f3221957df28bededadcbd9fc8a0661bde0/websockets-17.0.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:28012a54510fe8301bb893ef143cec30a2780a2d3bc20b7bbdf4379d7a63945d", size = 220573, upload-time = "2026-07-31T11:30:54.966Z" }, + { url = "https://files.pythonhosted.org/packages/1f/bd/ed48eca15725743ee7e2dc172e15c61de29e85ec98dace7b14257f366836/websockets-17.0.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22bd00f8bae2bccdb5dbe41e20f58ba44ca9fff0b4b561aaf39099c35da762ed", size = 221747, upload-time = "2026-07-31T11:30:56.762Z" }, + { url = "https://files.pythonhosted.org/packages/99/50/838deb7937a8225c4925dd4a977eafea473fabf444178a99de0bc7e92bb0/websockets-17.0.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e98ec9ec61cce5bc4b8b218322ad090b0994eb060bb04da704c62ef0a3d864e6", size = 223891, upload-time = "2026-07-31T11:30:58.127Z" }, + { url = "https://files.pythonhosted.org/packages/f8/e9/657fb70c6eb6bcd01adfa5d2b06496e9911e1c1a8813d353b8c00f7591cd/websockets-17.0.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8e387adb0c692c6b5571bdeafc8ac9d1901ea30f10309134780b16ecd35e6605", size = 222317, upload-time = "2026-07-31T11:30:59.416Z" }, + { url = "https://files.pythonhosted.org/packages/07/4c/82cb722afa5428fed981331210c4c07600570db01bb1620579f655b5adaf/websockets-17.0.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bd1470d2c53fe53269bf5619da7725d30dd9b9693f1689f7a85eab8dea734442", size = 221047, upload-time = "2026-07-31T11:31:00.757Z" }, + { url = "https://files.pythonhosted.org/packages/90/84/bd6d67d6bc65f0de0cb50de55dab42f256a9876a351c4736522eb168fda0/websockets-17.0.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:884af729b8ab50486acd94d9768c2b60914bf39b579ebba0a5cb73bfdfd61fd2", size = 218626, upload-time = "2026-07-31T11:31:02.48Z" }, + { url = "https://files.pythonhosted.org/packages/96/cb/6a372c8553976f0d8f97f5115826ed47e34b3be6b8bf0d0249af249a7416/websockets-17.0.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a60fa1a25cca1bcc2bf87b8d6be37a741f0a3239fb5e9cfb7a37173b68ffcf87", size = 221299, upload-time = "2026-07-31T11:31:03.795Z" }, + { url = "https://files.pythonhosted.org/packages/bc/31/f966e8472337974f74d788b3ef6c6f3b8b9a5f201efd16a843c91d269fa5/websockets-17.0.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:e8208f2729cba030ff872a92064c97584eeb9502f53d32a05a0f05d5a17ca6c6", size = 219789, upload-time = "2026-07-31T11:31:05.08Z" }, + { url = "https://files.pythonhosted.org/packages/57/34/404e83a6cc7b0efcac810b7041bffd72ff76900e6fd0aa45a26c92fb2ffe/websockets-17.0.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:54cdcaa56f5d3eafd57058f0fa4a3de93a310b43a3c4699f06efc4c0bd054a5a", size = 220678, upload-time = "2026-07-31T11:31:06.635Z" }, + { url = "https://files.pythonhosted.org/packages/e5/70/8946188c2a68d67251859b589a3634918cf7867bf0b891347a5ecaa43d30/websockets-17.0.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4d41c0a1d47a478bc432b3b9068097bee1ce0c5b19327ea6f75c2ab34ab1f2fb", size = 221697, upload-time = "2026-07-31T11:31:08.023Z" }, + { url = "https://files.pythonhosted.org/packages/04/16/ee73fc2083a2938ac6209f4ec804960496835b20a0068dbcfe8424957c04/websockets-17.0.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f991247276797d0c61ab7770bc9791eadc16f683b4d83517f624932adc1a8bab", size = 219390, upload-time = "2026-07-31T11:31:09.378Z" }, + { url = "https://files.pythonhosted.org/packages/94/6a/d5f88033c69932af6cdaa72da62516ade47c257e3bf69f4c0ba5f40e12a2/websockets-17.0.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:733e3cc7171fa1b899edbe725ef9382d0e960657dc1fd933f3281ae910c01dab", size = 220161, upload-time = "2026-07-31T11:31:10.915Z" }, + { url = "https://files.pythonhosted.org/packages/e1/2e/6183dd2c0370287ecf4afe0bb33aca364208e5e7b0e1a286adcaecc0c78b/websockets-17.0.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:810cb3fb5fa6e447216f4e82d9a85cb8aed0929ae3538153ddfe8a6e3121a58d", size = 220591, upload-time = "2026-07-31T11:31:12.289Z" }, + { url = "https://files.pythonhosted.org/packages/53/1a/d4437d3cb0691eeac2c6064e21c83a9eeda04f6ef0261abfc0dde708590d/websockets-17.0.1-pp311-pypy311_pp73-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a39ce3a7b0e6059be093213d637963101380157bcbad355916738fafb490698d", size = 211417, upload-time = "2026-07-31T11:31:20.687Z" }, + { url = "https://files.pythonhosted.org/packages/88/28/2d671e23a20359a1cc142848384659813b49028d46cb0acdc28e81ac59b5/websockets-17.0.1-pp311-pypy311_pp73-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2437d4ca208cc0f246d3a2297ae7474b4ba18261aaf5b9c79c84c031ecf348e1", size = 211309, upload-time = "2026-07-31T11:31:21.969Z" }, + { url = "https://files.pythonhosted.org/packages/02/52/b85a676b161991e5c0d884252376435f60510789e7740e3da73489d22a6e/websockets-17.0.1-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6740be6d1bab69f08ab52cb15b08f76c143b6fe61c580ba62bd929f3ab7a1d42", size = 212203, upload-time = "2026-07-31T11:31:23.38Z" }, + { url = "https://files.pythonhosted.org/packages/09/ce/3929538b2b9918f5eee623fbf3346893973191f6df93f19bbda097bd7bb7/websockets-17.0.1-py3-none-any.whl", hash = "sha256:c6be9cba65c65cc76dfa3d4619e359ff02a4476c74e179b215236c11a0b32345", size = 206718, upload-time = "2026-07-31T11:31:26.037Z" }, +] + [[package]] name = "widgetsnbextension" version = "4.0.15" @@ -4601,6 +7017,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/0e/fa3b193432cfc60c93b42f3be03365f5f909d2b3ea410295cf36df739e31/widgetsnbextension-4.0.15-py3-none-any.whl", hash = "sha256:8156704e4346a571d9ce73b84bee86a29906c9abfd7223b7228a28899ccf3366", size = 2196503, upload-time = "2025-11-01T21:15:53.565Z" }, ] +[[package]] +name = "xgrammar" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "pydantic" }, + { name = "torch" }, + { name = "transformers" }, + { name = "triton", marker = "platform_machine == 'x86_64'" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/81/f4/e71693d8cec60b7e36dab660784ecc5a6aa51e478a83b556011645c58c87/xgrammar-0.2.3.tar.gz", hash = "sha256:f76423630ae3ac4e090cb38ce1e30e7bcc69b3dee4d22d94353944386a4c6f18", size = 2447704, upload-time = "2026-06-27T04:45:24.759Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/b5/f1b54a6f652cbd9dead9a6e31c8eedeea846abae020fb7eb08cc90b13786/xgrammar-0.2.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48d2c9d2bab9b60653204bf334663e06f6044d8f5c104aca68ee23526afb3161", size = 44314487, upload-time = "2026-06-27T04:44:16.182Z" }, + { url = "https://files.pythonhosted.org/packages/84/f3/4bd6bd3dd9a0450d78bcd9db3593b5df9ab5baf62a8f50d265a933156c47/xgrammar-0.2.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d38fb3ad3118b8f08b1da53fb6feb81a206e6eb9df6abb16bda47e2cb272deef", size = 44855068, upload-time = "2026-06-27T04:44:18.898Z" }, + { url = "https://files.pythonhosted.org/packages/e4/79/0bb37937bf847c738c64b64dc50ddc12e7c526b34c5ab82cebe58da5ec8f/xgrammar-0.2.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:11255f184971489fc72b948b096e2917f482ba2dca975177f5411562cedb9c6d", size = 44314481, upload-time = "2026-06-27T04:44:28.875Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fd/5ebd5d14b8993cb225151bbb8f2011742fc7a7d94a3bdbc3ec3954b9b62d/xgrammar-0.2.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fdf081fab29694302d41d61dcf52fad7d253879a718bc6afc68db0a0dabd7f19", size = 44855110, upload-time = "2026-06-27T04:44:31.586Z" }, + { url = "https://files.pythonhosted.org/packages/4d/aa/1c74ff8bad6624c08a33924f7d051a9278a622b48d843e939d574a6b5bf8/xgrammar-0.2.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eeb5e46bd7d3230e5d8e6385793c48ec872a4fb377c19d341dbcaedc41f495e9", size = 44314496, upload-time = "2026-06-27T04:44:38.552Z" }, + { url = "https://files.pythonhosted.org/packages/28/f8/2407b44049416650c257e22399c32788d3497524a9a899addc74b5d27d1d/xgrammar-0.2.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:29d0fbd4709733b224e6e115d1001fb07124c71fe114a9087a2e3e53ce871517", size = 44855037, upload-time = "2026-06-27T04:44:41.122Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b5/983c482cad0a57230d16806b377710402644f7d4e6023fbf8897e0dc5256/xgrammar-0.2.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe71c621a18ec1fd0a74e755e72c5e521c4854b75d14ad52f1fc8c325d387124", size = 44314374, upload-time = "2026-06-27T04:44:51.526Z" }, + { url = "https://files.pythonhosted.org/packages/fb/9f/6c8601fa55545fdf9b9c95e289fc6db73b0c160759873f666a992741069d/xgrammar-0.2.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f26c8bb1845119856b09658bcf2ee525957dc618d954684e5c393d16bcc1f1da", size = 44855006, upload-time = "2026-06-27T04:44:54.545Z" }, + { url = "https://files.pythonhosted.org/packages/7b/b4/1b7cfc9c8d1bd3d33fcfd79d87be0bd959cb3b7418ddcf51c53d2d6ce35e/xgrammar-0.2.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1ba69a13ae4cc94b7a3a0b5ac67865ce372e0eac7b6486f7e8ca0c0cbbbc3097", size = 44314501, upload-time = "2026-06-27T04:45:03.955Z" }, + { url = "https://files.pythonhosted.org/packages/b5/a1/a5cc64b42a183ec89aeae5d9455017c17c5e8cc4597bbc30bf76fdb7ec40/xgrammar-0.2.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:838f88cd74c00670e4b0797ffd7bec8399b67f90ca0f199c96a68333f70553b4", size = 44854955, upload-time = "2026-06-27T04:45:06.557Z" }, +] + [[package]] name = "xxhash" version = "3.6.0" @@ -4813,3 +7257,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/b7/503c98092fb3b344a179579f55814b613c1fbb1c23b3ec14a7b008a66a6e/yarl-1.22.0-cp314-cp314t-win_arm64.whl", hash = "sha256:9f6d73c1436b934e3f01df1e1b21ff765cd1d28c77dfb9ace207f746d4610ee1", size = 85171, upload-time = "2025-10-06T14:12:16.935Z" }, { url = "https://files.pythonhosted.org/packages/73/ae/b48f95715333080afb75a4504487cbe142cae1268afc482d06692d605ae6/yarl-1.22.0-py3-none-any.whl", hash = "sha256:1380560bdba02b6b6c90de54133c81c9f2a453dee9912fe58c1dcced1edb7cff", size = 46814, upload-time = "2025-10-06T14:12:53.872Z" }, ] + +[[package]] +name = "z3-solver" +version = "4.15.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/8e/0c8f17309549d2e5cde9a3ccefa6365437f1e7bafe71878eaf9478e47b18/z3_solver-4.15.4.0.tar.gz", hash = "sha256:928c29b58c4eb62106da51c1914f6a4a55d0441f8f48a81b9da07950434a8946", size = 5018600, upload-time = "2025-10-29T18:12:03.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/c9/bb51a96af0091324c81b803f16c49f719f9f6ea0b0bb52200f5c97ec4892/z3_solver-4.15.4.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e103a6f203f505b8b8b8e5c931cc407c95b61556512d4921c1ddc0b3f41b08e", size = 29268352, upload-time = "2025-10-29T18:11:53.032Z" }, + { url = "https://files.pythonhosted.org/packages/bf/2e/0b49f7e4e53817cfb09a0f6585012b782dfe0b666e8abefcb4fac0570606/z3_solver-4.15.4.0-py3-none-manylinux_2_34_aarch64.whl", hash = "sha256:62c7e9cbdd711932301f29919ad9158de9b2f58b4d281dd259bbcd0a2f408ba1", size = 27226534, upload-time = "2025-10-29T18:11:55.59Z" }, +] From b2ff7a194472b1394ede604cdb85490b1b61da2d Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Mon, 10 Aug 2026 17:19:58 +0000 Subject: [PATCH 02/28] feat(dev): compile managed inference services Signed-off-by: Aaron Gonzales --- AGENTS.md | 2 +- README.md | 20 +- docs/concepts/inference-services.md | 238 +++++++ docs/concepts/local-vllm.md | 131 ---- docs/concepts/models.md | 2 +- docs/concepts/self-hosting-gliner.md | 68 +- .../posts/self-hosted-anonymizer-b300.md | 32 +- mkdocs.yml | 2 +- skills/anonymizer/SKILL.md | 4 +- skills/anonymizer/evals/evals.json | 2 +- tests/tools/test_inference_service.py | 660 ++++++++++++++++++ tests/tools/test_native_gliner.py | 276 ++++++++ tests/tools/test_vllm_debug.py | 184 ----- tools/inference_service.py | 20 + tools/inference_service_compiler/__init__.py | 5 + tools/inference_service_compiler/cli.py | 124 ++++ tools/inference_service_compiler/compiler.py | 330 +++++++++ tools/inference_service_compiler/models.py | 395 +++++++++++ .../native_gliner.py | 660 ++++++++++++++++++ tools/inference_service_compiler/runtime.py | 472 +++++++++++++ tools/serve_gliner.py | 486 ------------- tools/vllm_debug.py | 290 -------- 22 files changed, 3244 insertions(+), 1159 deletions(-) create mode 100644 docs/concepts/inference-services.md delete mode 100644 docs/concepts/local-vllm.md create mode 100644 tests/tools/test_inference_service.py create mode 100644 tests/tools/test_native_gliner.py delete mode 100644 tests/tools/test_vllm_debug.py create mode 100755 tools/inference_service.py create mode 100644 tools/inference_service_compiler/__init__.py create mode 100644 tools/inference_service_compiler/cli.py create mode 100644 tools/inference_service_compiler/compiler.py create mode 100644 tools/inference_service_compiler/models.py create mode 100755 tools/inference_service_compiler/native_gliner.py create mode 100644 tools/inference_service_compiler/runtime.py delete mode 100644 tools/serve_gliner.py delete mode 100644 tools/vllm_debug.py diff --git a/AGENTS.md b/AGENTS.md index f71af905..19fba613 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -123,4 +123,4 @@ make docs-serve # local MkDocs server at http://127.0.0.1:8000 For contributor workflow and branch naming see [CONTRIBUTING.md](CONTRIBUTING.md). For local setup, tests, docs, and day-to-day development tasks see [DEVELOPMENT.md](DEVELOPMENT.md). For code style and naming conventions see [STYLEGUIDE.md](STYLEGUIDE.md). -Reference GLiNER server for self-hosted detection: [`tools/serve_gliner.py`](tools/serve_gliner.py) — see [`docs/concepts/self-hosting-gliner.md`](docs/concepts/self-hosting-gliner.md). +Local GLiNER and vLLM services are compiled and managed through [`tools/inference_service.py`](tools/inference_service.py) — see [`docs/concepts/inference-services.md`](docs/concepts/inference-services.md) and [`docs/concepts/self-hosting-gliner.md`](docs/concepts/self-hosting-gliner.md). diff --git a/README.md b/README.md index 5afba45c..38350939 100644 --- a/README.md +++ b/README.md @@ -154,20 +154,22 @@ anonymizer --help # CLI usage make install-pre-commit # Install pre-commit hooks ``` -### Local vLLM debugging +### Local inference services -On a Linux GPU host, install the optional local-model dependency group and use -`tools/vllm_debug.py` to start or probe an OpenAI-compatible vLLM server: +Use the source-tree inference service compiler to create immutable plans and +managed launch receipts for native GLiNER or vLLM processes and containers: ```bash -uv sync --group dev --group local-models -uv run python tools/vllm_debug.py models --cached --json -uv run python tools/vllm_debug.py serve /path/to/cached/snapshot --served-model-name anonymizer-local -uv run python tools/vllm_debug.py models --endpoint http://127.0.0.1:8000/v1 +uv run tools/inference_service.py compile --intent intent.json --source-revision 3f68c145 --output plan.json +uv run tools/inference_service.py launch --plan plan.json --output launch.json +uv run tools/inference_service.py inspect --receipt launch.json +uv run tools/inference_service.py cancel --receipt launch.json ``` -The helper does not prefetch model weights. Use `--vllm-python /path/to/python` -when vLLM is installed in another virtual environment. The [local vLLM guide](docs/concepts/local-vllm.md) covers GPU-host requirements, Anonymizer configuration, verification, and internal-network access. +The tool is not part of the wheel and does not attach to externally owned +endpoints. The [local inference service guide](docs/concepts/inference-services.md) +covers typed intents, GPU-host setup, Docker, model discovery, capability +probes, and Anonymizer provider configuration. --- diff --git a/docs/concepts/inference-services.md b/docs/concepts/inference-services.md new file mode 100644 index 00000000..62022b46 --- /dev/null +++ b/docs/concepts/inference-services.md @@ -0,0 +1,238 @@ + + + +# Run Local Inference Services + +Anonymizer's source tree includes an inference service compiler for development +and controlled internal deployments. It compiles typed JSON intent into an +immutable plan before it starts a process or container. Launch, probe, inspect, +and cancel operations return versioned JSON receipts with the external identity +and known effects of the operation. + +The tool is source-owned under `tools/` and is not included in the +`nemo-anonymizer` wheel. It currently supports: + +- entity detection with the native NVIDIA GLiNER or GLiNER2 runtime; +- entity detection or generation through a vLLM-compatible model; +- managed local processes and managed local Docker containers; and +- direct HTTP access to the resulting OpenAI-compatible endpoint. + +It does not attach to an existing endpoint or manage remote compute. Configure +an existing provider URL directly in Anonymizer when another system owns that +service. + +## Lifecycle + +The CLI keeps compilation separate from runtime effects: + +```text +intent.json -> compile -> plan.json -> launch -> launch.json + inspect <-+ + cancel <-+ +``` + +Plans use the `inference-service.run-plan/v1` schema. Launch, capability-probe, +status, and cancellation receipts have their own v1 schemas. A SHA-256 digest +binds each plan to its exact intent, command, endpoint contract, compatibility +evidence, and source revision. Runtime commands reject a changed plan before +performing effects. + +## Native GLiNER + +Create `gliner-intent.json`: + +```json +{ + "schema_version": "inference-service.intent/v1", + "task": { + "kind": "entity-detection", + "dynamic_labels": true, + "offsets": true, + "scores": true + }, + "model": { + "kind": "hugging-face", + "model_id": "nvidia/gliner-pii" + }, + "engine": { + "kind": "native-gliner", + "family": "nvidia-gliner", + "device": "auto" + }, + "placement": { + "kind": "local-process", + "host": "127.0.0.1", + "port": 8001 + }, + "access": {"kind": "direct"}, + "lifecycle": { + "kind": "managed", + "startup_timeout_seconds": 120, + "shutdown_timeout_seconds": 30 + } +} +``` + +Compile and launch it from the repository root. Replace the example source +revision with the revision of your checkout: + +```bash +uv run tools/inference_service.py compile \ + --intent gliner-intent.json \ + --source-revision 3f68c145 \ + --output gliner-plan.json + +uv run tools/inference_service.py launch \ + --plan gliner-plan.json \ + --output gliner-launch.json +``` + +The launch returns only after `/v1/models` and an entity-detection contract +probe succeed. The receipt records the process ID plus its Linux start marker +when available, which lets a later invocation guard against PID reuse. + +To use GLiNER2, change the engine family to `gliner2` and select a compatible +checkpoint such as `fastino/gliner2-privacy-filter-PII-multi` in the model +field. `nvidia-gliner` and `nvidia/gliner-pii` remain the defaults. + +## Local vLLM Process + +On a Linux GPU host, install the optional source-tree dependency group: + +```bash +uv sync --group dev --group local-models +nvidia-smi +``` + +Create an intent with a generation task, vLLM engine, and local-process +placement: + +```json +{ + "schema_version": "inference-service.intent/v1", + "task": {"kind": "generation", "chat": true}, + "model": { + "kind": "hugging-face", + "model_id": "openai/gpt-oss-20b", + "revision": "PIN_A_MODEL_REVISION_HERE" + }, + "engine": { + "kind": "vllm", + "executable": ".venv/bin/vllm", + "served_model_name": "anonymizer-local", + "gpu_memory_utilization": 0.85, + "max_model_len": 8192 + }, + "placement": { + "kind": "local-process", + "host": "127.0.0.1", + "port": 8000 + }, + "access": {"kind": "direct"}, + "lifecycle": { + "kind": "managed", + "startup_timeout_seconds": 600, + "shutdown_timeout_seconds": 30 + } +} +``` + +Use the same `compile` and `launch` commands shown for GLiNER. The model ID may +cause vLLM to download weights. List existing Hugging Face cache snapshots +without downloading anything: + +```bash +uv run tools/inference_service.py models --output cached-models.json +``` + +Add a LoRA artifact to the model when needed: + +```json +"adapter": { + "path": "/models/privacy-adapter", + "name": "privacy" +} +``` + +The compiler renders the corresponding vLLM `--lora-modules` arguments. + +## Docker vLLM + +Change the placement to Docker to use vLLM's official OpenAI-compatible image: + +```json +"placement": { + "kind": "docker", + "host": "127.0.0.1", + "port": 8000, + "image": "vllm/vllm-openai:v0.20.0", + "runtime": "docker", + "gpus": "all", + "hugging_face_cache": "/home/user/.cache/huggingface" +} +``` + +The plan records the exact image and complete `docker run` argv. Launch receipts +record the container ID, and `inspect` and `cancel` reconnect through that ID. +Pin an image version appropriate for the host's driver and CUDA compatibility. + +## Secrets + +Set `api_key_env` on the vLLM engine to reference a named environment variable: + +```json +"api_key_env": "LOCAL_VLLM_API_KEY" +``` + +Plans serialize only that source name and render the service environment value +as ``. `launch` maps it to `VLLM_API_KEY` without +putting the value in process arguments or Docker command metadata. Docker uses +`--env VLLM_API_KEY` to inherit the resolved value. Launch fails before starting +the process or container when the source variable is absent. Do not commit local +secret files. + +## Inspect, Probe, and Cancel + +Use the plan to collect a fresh capability receipt, or the launch receipt to +inspect and stop the managed service: + +```bash +uv run tools/inference_service.py probe \ + --plan gliner-plan.json \ + --output gliner-probe.json + +uv run tools/inference_service.py inspect \ + --receipt gliner-launch.json \ + --output gliner-status.json + +uv run tools/inference_service.py cancel \ + --receipt gliner-launch.json \ + --output gliner-cancellation.json +``` + +Local-process logs are written under `.inference-service-runs/` by default. +Use `launch --log-directory PATH` to select another location. + +## Connect Anonymizer + +Both native GLiNER and vLLM expose OpenAI-compatible URLs. Add the compiled +endpoint to a custom provider file: + +```yaml title="providers.yaml" +providers: + - name: local-inference + endpoint: http://127.0.0.1:8000/v1 + provider_type: openai + api_key: EMPTY +``` + +Set the selected model configuration's `provider` to `local-inference` and its +`model` to the served model name. Custom `model_configs` replaces Anonymizer's +entire bundled model pool, so retain every alias required by the roles you use. +See [Custom models](models.md#custom-models) for the role map and validation +command. + +Compilation proves only static compatibility. The launch probe proves the +observed endpoint shape. Neither proves that a model meets your privacy or +utility requirements; run Anonymizer preview and evaluation before trusting a +new model or engine combination. diff --git a/docs/concepts/local-vllm.md b/docs/concepts/local-vllm.md deleted file mode 100644 index 884ed208..00000000 --- a/docs/concepts/local-vllm.md +++ /dev/null @@ -1,131 +0,0 @@ - - - -# Run Local vLLM Models - -This guide is for contributors who operate an Anonymizer source checkout on a -Linux host with NVIDIA GPUs. It starts a local, OpenAI-compatible [vLLM](https://docs.vllm.ai/) -endpoint for development, evaluation, or an internal deployment. It does not -download a model or configure production networking for you. - -The helper script, [`tools/vllm_debug.py`](https://github.com/NVIDIA-NeMo/Anonymizer/blob/main/tools/vllm_debug.py), is a source-tree development tool. It is not included in the `nemo-anonymizer` package. - -## Host Requirements - -Before installing the local-model dependencies, confirm that the host has: - -- Linux, Python 3.11 or later, and an NVIDIA GPU that has enough free VRAM for the selected model and its context length. -- An NVIDIA driver that can communicate with the GPU. Verify this with `nvidia-smi`. -- Access to the model weights, either through the Hugging Face cache or a local model directory. Observe the model license and access controls. - -The dependency group installs vLLM and its CUDA-compatible dependencies. Driver, GPU, model-size, and CUDA compatibility remain properties of the host. Start with a small model and a short context length before adopting a model for a benchmark or workflow. - -## Install the Local-Model Environment - -From a source checkout, install the development and local-model groups: - -```bash -uv sync --group dev --group local-models -nvidia-smi -uv run python -c 'import torch; assert torch.cuda.is_available(); print(torch.cuda.get_device_name(0))' -``` - -If vLLM is kept in a separate virtual environment, the helper can use that interpreter with `--vllm-python /path/to/python`. This is useful when the model-serving environment has different CUDA constraints from the Anonymizer development environment. - -## Select a Cached Model - -The helper discovers snapshots already in the Hugging Face hub cache. It does not fetch model weights: - -```bash -uv run python tools/vllm_debug.py models --cached --json -``` - -Pass a `snapshot_path` from that output to `serve`. A Hugging Face model ID is also accepted by vLLM, but may trigger a download. - -## Start and Verify a Server - -Run the server from a terminal that will remain open: - -```bash -uv run python tools/vllm_debug.py serve /path/to/cached/snapshot \ - --served-model-name anonymizer-local \ - --gpu-memory-utilization 0.85 \ - --max-model-len 8192 -``` - -The default bind address, `127.0.0.1:8000`, keeps the endpoint local to the GPU host. `--served-model-name` gives clients a stable model ID, independent of the cache directory name. Stop the server with `Ctrl-C`. - -From another terminal, verify both the model registration and one completion: - -```bash -uv run python tools/vllm_debug.py models -uv run python tools/vllm_debug.py call \ - --model anonymizer-local \ - --prompt 'Reply with the word ready.' \ - --timeout-seconds 120 -``` - -Use `--dry-run` with `serve` to print the vLLM command before reserving GPU memory. For multi-GPU models, add `--tensor-parallel-size N`. For a LoRA adapter, add `--adapter /path/to/adapter` and, if needed, `--adapter-name NAME`. - -## Connect Anonymizer - -vLLM presents an OpenAI-compatible endpoint. Add it to a custom provider file: - -```yaml title="providers.yaml" -providers: - - name: local-vllm - endpoint: http://127.0.0.1:8000/v1 - provider_type: openai - api_key: EMPTY # vLLM has no API key unless one is configured below -``` - -In a custom `models.yaml`, set each local model configuration's `provider` to `local-vllm` and its `model` to the server's served-model name, `anonymizer-local` in the example above. Then pass both files to `Anonymizer`: - -```python -from anonymizer import Anonymizer - -anonymizer = Anonymizer( - model_providers="providers.yaml", - model_configs="models.yaml", -) -``` - -`model_configs` replaces Anonymizer's entire bundled model pool. Copy the bundled [`models.yaml`](https://github.com/NVIDIA-NeMo/Anonymizer/blob/main/src/anonymizer/config/default_model_configs/models.yaml), retain every alias required by the roles you use, and change only the models that should route to vLLM. See [Custom models](models.md#custom-models) for the role map and validation command. - -Use `anonymizer.validate_config(config)` before processing data. A successful HTTP probe only confirms that the server responds. It does not establish that a model is suitable for detection, replacement, or privacy-preserving rewrite quality. - -## Access From Another Host - -Keep the default loopback binding whenever Anonymizer and vLLM run on the same machine. If a trusted internal client needs the endpoint, use a network policy or firewall as well as a server API key: - -```bash -# .env.local is ignored by this repository. Do not commit it. -LOCAL_VLLM_API_KEY='replace-with-a-long-random-secret' - -# Load it in the shell that starts the server and the client. -set -a; source .env.local; set +a - -uv run python tools/vllm_debug.py serve /path/to/cached/snapshot \ - --host 0.0.0.0 \ - --served-model-name anonymizer-local \ - --api-key-env LOCAL_VLLM_API_KEY -``` - -Set the same secret's environment-variable name in the Anonymizer provider configuration: - -```yaml title="providers.yaml" -providers: - - name: local-vllm - endpoint: http://gpu-host.internal:8000/v1 - provider_type: openai - api_key: LOCAL_VLLM_API_KEY -``` - -Use an ignored `.env.local` file, or an ignored `.mise.local.toml` file when your checkout uses Mise, to keep local endpoint credentials out of version control. Load the secret only in the shell or task runner that starts the server and client. Do not expose a raw vLLM endpoint to the public internet. For production access, place it behind your organization's authenticated TLS-enabled network boundary. - -## Operating Notes - -- List the server's registered model IDs with `models` after every model or adapter change. Client `model` values must match one of those IDs. -- Start conservatively with `--gpu-memory-utilization` and `--max-model-len`; increase them only after observing stable GPU memory use and latency. -- Keep GLiNER separate from the LLM when GPU memory is constrained. The [self-hosted GLiNER guide](self-hosting-gliner.md) describes the detection endpoint. -- Treat local-model output as untrusted until it has passed the same Anonymizer preview, evaluation, and privacy review used for any other provider. diff --git a/docs/concepts/models.md b/docs/concepts/models.md index cb92eec0..700294f1 100644 --- a/docs/concepts/models.md +++ b/docs/concepts/models.md @@ -36,7 +36,7 @@ Each pipeline stage has a **role** mapped to one of these aliases. See the full Pass `model_providers` when you need a non-default endpoint — for example OpenAI, OpenRouter, a local GLiNER server, or an internal inference deployment. Plain `Anonymizer()` already uses bundled [build.nvidia.com](https://build.nvidia.com) settings; override only when your models point at a different provider name or URL. -For a GPU-hosted OpenAI-compatible LLM endpoint, see [Run local vLLM models](local-vllm.md). That guide covers the source-tree helper, optional dependencies, and a `local-vllm` provider configuration. +For managed native GLiNER and vLLM endpoints, see [Run local inference services](inference-services.md). That guide covers immutable plans, local processes, Docker, capability receipts, and provider configuration. Set your API keys first: diff --git a/docs/concepts/self-hosting-gliner.md b/docs/concepts/self-hosting-gliner.md index 9e7cc42f..56dbf26f 100644 --- a/docs/concepts/self-hosting-gliner.md +++ b/docs/concepts/self-hosting-gliner.md @@ -5,9 +5,9 @@ By default, Anonymizer's entity detection stage calls the hosted `nvidia/gliner-pii` model on `build.nvidia.com`. For PHI-sensitive workloads that cannot leave the host, or latency-critical setups, you can serve GLiNER locally instead. -The model is small (~500 MB) and runs comfortably on CPU — making it a good fit to run alongside a local LLM without competing for GPU memory. It also runs on GPU if one is available, which cuts detection latency on long documents. +The default NVIDIA GLiNER model is small (~500 MB) and runs comfortably on CPU — making it a good fit to run alongside a local LLM without competing for GPU memory. It also runs on GPU if one is available, which cuts detection latency on long documents. The optional GLiNER2 PII model is also fully local and supports GPU or CPU inference. -The reference server script (`tools/serve_gliner.py`) is **not** installed with `pip install nemo-anonymizer` — get it from a source checkout of this repository (see **Running it** below). +The characterized native server lives inside the source-tree [inference service compiler](inference-services.md). It is **not** installed with `pip install nemo-anonymizer`; compile and launch it from a source checkout. --- @@ -52,24 +52,17 @@ Long inputs are split into overlapping chunks before inference. A self-hosted se ## Reference implementation -A minimal FastAPI reference server at [`tools/serve_gliner.py`](https://github.com/NVIDIA-NeMo/Anonymizer/blob/main/tools/serve_gliner.py) in the Anonymizer GitHub repository implements the contract above. It loads `nvidia/gliner-pii`, exposes `POST /v1/chat/completions` (and `GET /v1/models`), and uses two levels of batching: +The native GLiNER runtime compiled by `tools/inference_service.py` implements the contract above. Its internal server module defaults to `nvidia/gliner-pii`, also supports the local PII-capable `fastino/gliner2-privacy-filter-PII-multi` model, exposes `POST /v1/chat/completions` (and `GET /v1/models`), and uses two levels of batching: -1. **Chunk batching** — long text is split into overlapping windows; all chunks are passed to one `model.inference(...)` call. +1. **Chunk batching** — long text is split into overlapping windows; all chunks are passed to one runtime batch call. 2. **Request coalescing** (optional, on by default) — concurrent HTTP requests from DataDesigner are grouped briefly, then all their chunks are inferred together. -```python title="tools/serve_gliner.py (excerpt)" -@app.post("/v1/chat/completions") +```python title="tools/inference_service_compiler/native_gliner.py (excerpt)" +@api.post("/v1/chat/completions") async def chat_completions(request: Request): - body = await request.json() - text = _extract_text(body.get("messages", [])) - params = DetectParams( - labels=tuple(body.get("labels") or []), - threshold=float(body.get("threshold", 0.3)), - chunk_length=int(body.get("chunk_length", 384)), - overlap=int(body.get("overlap", 128)), - flat_ner=bool(body.get("flat_ner", False)), - inference_batch_size=int(body.get("batch_size", 8)), - ) + body = require_mapping(await request.json(), "request") + params = parse_detect_params(body) + text = extract_text(body.get("messages", [])) entities = await detector.detect(text, params) ... ``` @@ -91,42 +84,30 @@ Set `GLINER_BATCH_MODE=false` to disable request coalescing; chunk batching stil !!! note "Source checkout only" - `tools/serve_gliner.py` ships in the [Anonymizer GitHub repository](https://github.com/NVIDIA-NeMo/Anonymizer), not in the `nemo-anonymizer` wheel. Clone the repo or download the file, then run it from that tree: - - ```bash - git clone https://github.com/NVIDIA-NeMo/Anonymizer.git - cd Anonymizer - pip install fastapi uvicorn gliner - python tools/serve_gliner.py - ``` + `tools/inference_service.py` ships in the [Anonymizer GitHub repository](https://github.com/NVIDIA-NeMo/Anonymizer), not in the `nemo-anonymizer` wheel. Clone the repository and run the compiler from its root. ### Dependencies -```bash -pip install fastapi uvicorn gliner -# or with uv -uv pip install fastapi uvicorn gliner -``` +The managed native server is a [PEP 723](https://peps.python.org/pep-0723/) uv script and declares its own Python 3.13+ dependencies. Install [uv](https://docs.astral.sh/uv/); launch resolves the isolated environment for the selected local runtime. The first environment setup can be large because the runtime packages include Torch and its platform dependencies. No package installation in the Anonymizer environment is required. -On first launch the `gliner` package will download `nvidia/gliner-pii` from HuggingFace and cache it under `~/.cache/huggingface/`. No HuggingFace token is required (public model). +On first launch, the selected public checkpoint is downloaded from Hugging Face and cached under `~/.cache/huggingface/`. No Hugging Face token is required. Package and checkpoint setup use the network; inference stays local and the server does not call a remote inference service after setup. ### Start the server -```bash -python tools/serve_gliner.py -# INFO Uvicorn running on http://127.0.0.1:8001 +Create a native GLiNER intent, compile it, and launch the resulting plan as +shown in [Run local inference services](inference-services.md#native-gliner). +The intent keeps the model checkpoint, engine family, device, placement, +access, and managed lifecycle separate. `nvidia-gliner` is the default engine +family and `nvidia/gliner-pii` is the default model; GLiNER2 uses the +`fastino/gliner2-privacy-filter-PII-multi` checkpoint. -# Optional: override port (default: 8001) -python tools/serve_gliner.py --port 9000 +Launch writes a versioned receipt only after the model-list and detection +contract probes pass. Use that receipt with the compiler's `inspect` and +`cancel` commands instead of supervising the internal server module directly. -# Optional: listen on all interfaces — no auth; use only on trusted networks -python tools/serve_gliner.py --host 0.0.0.0 +The model families do not use identical label vocabularies. The request example below targets the default NVIDIA model and uses `user_name`; the default GLiNER2 PII checkpoint uses `username` for that category. -# Optional: pick device explicitly (auto prefers mps, then cuda, then cpu) -DEVICE=cuda python tools/serve_gliner.py -``` - -The reference server has **no authentication**. The default bind address is `127.0.0.1` so detection traffic stays on localhost. Use `--host 0.0.0.0` only when Anonymizer runs on another host in a trusted environment. +The reference server has **no authentication**. The default bind address is `127.0.0.1` so detection traffic stays on localhost. Use `--host 0.0.0.0` only when Anonymizer runs on another host in a trusted environment, ideally behind authentication and TLS termination. Verify the server is reachable: @@ -195,7 +176,6 @@ model_configs: - alias: gliner-pii-detector model: nvidia/gliner-pii provider: local-gliner - skip_health_check: true # the default health check sends no `labels`, which GLiNER can't handle inference_parameters: max_parallel_requests: 8 # send concurrent rows; the reference server batches them timeout: 120 @@ -230,8 +210,6 @@ anonymizer = Anonymizer( ) ``` -Set `skip_health_check: true` on the detector alias: Anonymizer's default probe sends `prompt="Hello!"` with no `labels` field, which is not a valid GLiNER request. - --- ## Performance notes diff --git a/docs/devnotes/posts/self-hosted-anonymizer-b300.md b/docs/devnotes/posts/self-hosted-anonymizer-b300.md index f20e26ca..92bb9c39 100644 --- a/docs/devnotes/posts/self-hosted-anonymizer-b300.md +++ b/docs/devnotes/posts/self-hosted-anonymizer-b300.md @@ -133,16 +133,32 @@ The `CUDA_ROOT` path above is specific to the Brev B300 SXM6 environment used fo `--gpu-memory-utilization 0.45` was a conservative co-location setting, not a compute throttle. In vLLM it controls the GPU memory budget for model weights and KV cache. Qwen could use more memory if the run needed a larger KV cache, but this setting left headroom for the GLiNER server on the same GPU and still completed the measured batches with zero failures. -GLiNER ran on the same machine. The command uses `tools/serve_gliner.py`, the reference GLiNER server from an Anonymizer source checkout; see [Self-hosting GLiNER](../../concepts/self-hosting-gliner.md) for the server contract and setup details. +GLiNER ran on the same machine. Current reruns use the source-tree inference +service compiler described in [Self-hosting GLiNER](../../concepts/self-hosting-gliner.md), +which records the exact model, engine, placement, batch environment, endpoint, +and process identity in versioned plans and receipts. + +```json title="gliner-b300-intent.json" +{ + "schema_version": "inference-service.intent/v1", + "task": {"kind": "entity-detection", "dynamic_labels": true, "offsets": true, "scores": true}, + "model": {"kind": "hugging-face", "model_id": "nvidia/gliner-pii", "revision": "bd23e8ef4425fd04e34c5204ab49ffaa706eae79"}, + "engine": {"kind": "native-gliner", "family": "nvidia-gliner", "device": "cuda", "max_batch_requests": 64, "batch_wait_ms": 10}, + "placement": {"kind": "local-process", "host": "127.0.0.1", "port": 9000}, + "access": {"kind": "direct"}, + "lifecycle": {"kind": "managed", "startup_timeout_seconds": 300, "shutdown_timeout_seconds": 30} +} +``` ```bash -mkdir -p logs - -DEVICE=cuda \ -GLINER_MAX_BATCH_REQUESTS=64 \ -GLINER_BATCH_WAIT_MS=10 \ -nohup .venv/bin/python tools/serve_gliner.py --port 9000 \ - > logs/gliner.log 2>&1 & +uv run tools/inference_service.py compile \ + --intent gliner-b300-intent.json \ + --source-revision 3f68c145 \ + --output gliner-b300-plan.json +uv run tools/inference_service.py launch \ + --plan gliner-b300-plan.json \ + --output gliner-b300-launch.json \ + --log-directory logs ``` For archival reruns, pin the GLiNER model as well. The server default used here is `nvidia/gliner-pii`, which Hugging Face resolved as `nvidia/gliner-PII` revision `bd23e8ef4425fd04e34c5204ab49ffaa706eae79` as of this write-up; serving a newer detector snapshot can change entity counts. diff --git a/mkdocs.yml b/mkdocs.yml index 1d047a8d..e2e95235 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -161,7 +161,7 @@ nav: - Choosing a Strategy: concepts/choosing-a-strategy.md - Evaluation: concepts/evaluation.md - Self-hosting GLiNER: concepts/self-hosting-gliner.md - - Run Local vLLM Models: concepts/local-vllm.md + - Run Local Inference Services: concepts/inference-services.md - Troubleshooting: troubleshooting.md - Tutorials: - Overview: tutorials/index.md diff --git a/skills/anonymizer/SKILL.md b/skills/anonymizer/SKILL.md index 30642118..9fef5d25 100644 --- a/skills/anonymizer/SKILL.md +++ b/skills/anonymizer/SKILL.md @@ -48,7 +48,7 @@ regulatory and business context. - **`risk_tolerance` only applies to Rewrite mode**, not Replace. - **`PrivacyGoal.protect` and `.preserve` must each be 10–1000 chars and at least 3 words.** Be specific (categories, named identifiers, structural facets); avoid generic phrasing like "preserve meaning". - **Validator pool is the only model role with built-in load-spreading.** Set `entity_validator: [a, b, c]` in `models.yaml` if rate limits drop rows. Other roles (rewriter, evaluator, etc.) are single-alias. -- **Self-hosted GLiNER:** When detection must not call `build.nvidia.com` (PHI on-prem, air-gapped, latency), run the reference server from a **source checkout** with `python tools/serve_gliner.py`. The server is not installed by `pip install nemo-anonymizer`. Add a provider with `endpoint: http://localhost:8001/v1`, then route `entity_detector` through a `gliner-pii-detector` alias with `provider: local-gliner` and `skip_health_check: true`. Match any custom `--port` or `--host` in the provider endpoint. `model_configs` is a **complete** model pool, not an overlay. Copy `src/anonymizer/config/default_model_configs/models.yaml` and change only the detector entry, keeping `gpt-oss-120b` and `nemotron-30b-thinking`. See [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). +- **Self-hosted GLiNER:** When detection must not call `build.nvidia.com` (PHI on-prem, air-gapped, latency), use `tools/inference_service.py` from a **source checkout** to compile a native-GLiNER intent and launch its managed plan. The tool and native server are not installed by `pip install nemo-anonymizer`. Add a provider with `endpoint: http://localhost:8001/v1`, then route `entity_detector` through a `gliner-pii-detector` alias with `provider: local-gliner` and `skip_health_check: true`. Match the compiled host and port in the provider endpoint. `model_configs` is a **complete** model pool, not an overlay. Copy `src/anonymizer/config/default_model_configs/models.yaml` and change only the detector entry, keeping `gpt-oss-120b` and `nemotron-30b-thinking`. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). - **The evaluation judges use their own model roles** (`entity_coverage_judge`, `detection_validity_judge`, `replace_type_fidelity_judge`, `replace_relational_consistency_judge`, `replace_attribute_fidelity_judge`, `rewrite_judge`), configured in the `evaluate` section of `models.yaml`. They are **not** consumed by `preview()` / `run()`, so a config that anonymizes fine can still fail validation at `evaluate()` if those roles are unset. Defaults ship in `src/anonymizer/config/default_model_configs/evaluate.yaml` (`entity_coverage_judge` defaults to `nemotron-super`). - **Verdict columns are null when the judge was unavailable** — `None` means "unscored", never a pass. `entity_coverage` is a `0–1` float (`1.0` = no missed candidate values or no PII found) or `None`; `missed_entities` lists unique candidate values the anonymizer failed to detect. Replace verdict columns (`type_fidelity_valid`, etc.) are `True` / `False` / `None`. Rewrite `detection_valid` is a `0–1` float fraction (or `None` if unscored). Inspect verdicts per record with `evaluated.display_record(i)`. - **`EvaluateConfig` has one knob today: `compute_detection_validity`** (default `False`). Plain `anonymizer.evaluate(result)` runs entity coverage + the mode's quality judges; pass `EvaluateConfig(compute_detection_validity=True)` only to additionally score detection validity (an internal-facing tag-precision metric). @@ -73,7 +73,7 @@ read `docs/troubleshooting.md` or the - **`anonymizer` not installed:** Tell the user `nemo-anonymizer` is not in this Python environment (requires Python ≥ 3.11). Ask if they want you to install it (`pip install nemo-anonymizer`) or do it themselves. Do not install without permission. - **Model/provider setup:** Plain `Anonymizer()` ships with bundled `models.yaml` and `providers.yaml` (see `src/anonymizer/config/default_model_configs/`). For the default path, confirm `NVIDIA_API_KEY` is set. Pass custom `model_configs` or `model_providers` only for non-default endpoints or model pools. See `docs/concepts/models.md` or the [published models guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/models/). - **LLM calls failing at preview:** Check for a missing or invalid API key, a network problem, or a wrong endpoint URL. See `docs/troubleshooting.md` "Validation passed but `preview` errors at LLM call" or the [published troubleshooting guide](https://nvidia-nemo.github.io/Anonymizer/dev/troubleshooting/). -- **Local / on-prem GLiNER:** Clone or download `tools/serve_gliner.py` from the Anonymizer repo, start the server, add a provider with `endpoint: http://localhost:8001/v1`, and point `gliner-pii-detector` at `provider: local-gliner` with `skip_health_check: true`. Preflight errors about missing aliases usually mean `model_configs` lists only the detector. Include the full default pool. A wrong endpoint or stopped server surfaces as a detection failure during preview. See [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). +- **Local / on-prem GLiNER:** Clone the Anonymizer repository, compile and launch a native-GLiNER intent with `tools/inference_service.py`, add a provider with the plan's endpoint (normally `http://localhost:8001/v1`), and point `gliner-pii-detector` at `provider: local-gliner` with `skip_health_check: true`. Preflight errors about missing aliases usually mean `model_configs` lists only the detector. Include the full default pool. Use the launch receipt to inspect or cancel the managed process. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). # Output Template diff --git a/skills/anonymizer/evals/evals.json b/skills/anonymizer/evals/evals.json index 277b61fa..645bc2b8 100644 --- a/skills/anonymizer/evals/evals.json +++ b/skills/anonymizer/evals/evals.json @@ -53,7 +53,7 @@ "expected_skill": "anonymizer", "should_trigger": true, "expected_script": null, - "ground_truth": "The agent loads the anonymizer skill and explains the self-hosted GLiNER path: run tools/serve_gliner.py from a source checkout, add an OpenAI-compatible provider at http://localhost:8001/v1, point the gliner-pii-detector/entity_detector alias at provider local-gliner with skip_health_check true, and keep model_configs as a complete model pool copied from defaults rather than a partial overlay.", + "ground_truth": "The agent loads the anonymizer skill and explains the self-hosted GLiNER path: use tools/inference_service.py from a source checkout to compile a native-GLiNER intent and launch its managed plan, add an OpenAI-compatible provider at the plan endpoint (normally http://localhost:8001/v1), point the gliner-pii-detector/entity_detector alias at provider local-gliner with skip_health_check true, and keep model_configs as a complete model pool copied from defaults rather than a partial overlay.", "expected_behavior": [ "The agent read skills/anonymizer/SKILL.md before answering", "The answer says the reference GLiNER server comes from a source checkout, not pip-installed package files", diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py new file mode 100644 index 00000000..2ae834f1 --- /dev/null +++ b/tests/tools/test_inference_service.py @@ -0,0 +1,660 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Behavior tests for the source-tree inference service compiler.""" + +from __future__ import annotations + +import importlib +import json +import stat +import sys +from pathlib import Path +from types import ModuleType +from typing import Any +from unittest import mock + +import httpx +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] +CLI_PATH = REPO_ROOT / "tools" / "inference_service.py" +TOOLS_ROOT = REPO_ROOT / "tools" +NATIVE_GLINER_PATH = TOOLS_ROOT / "inference_service_compiler" / "native_gliner.py" +REMOVED_GLINER_PATH = TOOLS_ROOT / "serve_gliner.py" + + +def load_compiler_modules() -> tuple[ModuleType, ModuleType]: + """Load source-tree modules through the same import root as the CLI.""" + sys.path.insert(0, str(TOOLS_ROOT)) + try: + models = importlib.import_module("inference_service_compiler.models") + compiler = importlib.import_module("inference_service_compiler.compiler") + finally: + sys.path.pop(0) + return models, compiler + + +def load_cli_module() -> ModuleType: + """Load the CLI through the source-tree import root.""" + sys.path.insert(0, str(TOOLS_ROOT)) + try: + return importlib.import_module("inference_service_compiler.cli") + finally: + sys.path.pop(0) + + +def load_runtime_module() -> ModuleType: + """Load the runtime through the source-tree import root.""" + sys.path.insert(0, str(TOOLS_ROOT)) + try: + return importlib.import_module("inference_service_compiler.runtime") + finally: + sys.path.pop(0) + + +def build_generation_plan( + models: ModuleType, + compiler: ModuleType, + *, + api_key_env: str | None = None, + docker: bool = False, +) -> Any: + """Build one local vLLM plan without runtime effects.""" + intent = models.InferenceIntent( + task=models.Generation(chat=True), + model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b", revision="abcdef0123456789"), + engine=models.VllmEngine(api_key_env=api_key_env), + placement=( + models.DockerPlacement( + host="127.0.0.1", + port=8000, + image="vllm/vllm-openai:v0.20.0", + gpus="all", + ) + if docker + else models.LocalProcessPlacement(host="127.0.0.1", port=8000) + ), + access=models.DirectAccess(), + lifecycle=models.ManagedLifecycle(startup_timeout_seconds=30), + ) + return compiler.compile_intent(intent, source_revision="3f68c145") + + +def test_cli_is_a_directly_executable_source_tree_entrypoint() -> None: + """The compiler has one stable CLI without creating an installable package.""" + assert CLI_PATH.is_file() + assert CLI_PATH.stat().st_mode & stat.S_IXUSR + + +def test_compile_native_gliner_local_process_plan() -> None: + """Native detection compiles into a deterministic effect-free local plan.""" + models, compiler = load_compiler_modules() + assert callable(getattr(compiler, "compile_intent", None)) + + intent = models.InferenceIntent( + task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), + model=models.HuggingFaceModel(model_id="nvidia/gliner-pii", revision="0123456789abcdef"), + engine=models.NativeGlinerEngine(family="nvidia-gliner", device="cpu"), + placement=models.LocalProcessPlacement(host="127.0.0.1", port=8001), + access=models.DirectAccess(), + lifecycle=models.ManagedLifecycle(startup_timeout_seconds=120), + ) + + first = compiler.compile_intent(intent, source_revision="3f68c145") + second = compiler.compile_intent(intent, source_revision="3f68c145") + + assert first == second + assert first.schema_version == "inference-service.run-plan/v1" + assert first.intent_digest == compiler.digest_model(intent) + assert first.plan_digest == compiler.digest_plan(first) + assert first.endpoint.url == "http://127.0.0.1:8001/v1" + assert first.readiness.url == "http://127.0.0.1:8001/v1/models" + assert first.expected_model == "nvidia/gliner-pii" + assert first.declared_capabilities == ("dynamic-labels", "offsets", "scores") + assert first.required_capabilities == ("dynamic-labels", "offsets", "scores") + assert first.runtime.kind == "local-process" + assert first.command.render_argv() == ( + "uv", + "run", + "--script", + "tools/inference_service_compiler/native_gliner.py", + "--host", + "127.0.0.1", + "--port", + "8001", + "--model", + "nvidia-gliner", + "--checkpoint", + "nvidia/gliner-pii", + "--revision", + "0123456789abcdef", + ) + + with pytest.raises(Exception, match="frozen"): + first.endpoint.port = 9000 + + +def test_compile_vllm_docker_plan_keeps_secrets_symbolic() -> None: + """Docker plans pin the image and never serialize an API-key value.""" + models, compiler = load_compiler_modules() + intent = models.InferenceIntent( + task=models.Generation(chat=True), + model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b", revision="abcdef0123456789"), + engine=models.VllmEngine( + served_model_name="anonymizer-local", + api_key_env="LOCAL_VLLM_API_KEY", + tensor_parallel_size=2, + gpu_memory_utilization=0.8, + max_model_len=4096, + eager=True, + ), + placement=models.DockerPlacement( + host="127.0.0.1", + port=8000, + image="vllm/vllm-openai:v0.20.0", + gpus="all", + ), + access=models.DirectAccess(), + lifecycle=models.ManagedLifecycle(startup_timeout_seconds=300), + ) + + plan = compiler.compile_intent(intent, source_revision="3f68c145") + rendered = plan.model_dump_json() + + assert plan.runtime.kind == "docker" + assert plan.runtime.image == "vllm/vllm-openai:v0.20.0" + assert plan.endpoint.url == "http://127.0.0.1:8000/v1" + assert plan.expected_model == "anonymizer-local" + assert plan.required_capabilities == ("chat-completions",) + assert plan.declared_capabilities == ("chat-completions",) + assert "LOCAL_VLLM_API_KEY" in rendered + assert "test-secret" not in rendered + assert "test-secret" not in plan.command.render_argv() + assert plan.command.render_environment() == {"VLLM_API_KEY": ""} + assert plan.command.render_environment(resolve_secrets={"LOCAL_VLLM_API_KEY": "test-secret"}) == { + "VLLM_API_KEY": "test-secret" + } + assert plan.command.render_argv()[:6] == ( + "docker", + "run", + "--detach", + "--rm", + "--gpus", + "all", + ) + environment_index = plan.command.render_argv().index("--env") + assert plan.command.render_argv()[environment_index : environment_index + 2] == ("--env", "VLLM_API_KEY") + revision_index = plan.command.render_argv().index("--revision") + assert plan.command.render_argv()[revision_index : revision_index + 4] == ( + "--revision", + "abcdef0123456789", + "--tokenizer-revision", + "abcdef0123456789", + ) + + +def test_compiler_rejects_unsupported_native_generation() -> None: + """Compatibility failures are typed compiler diagnostics, not runtime surprises.""" + models, compiler = load_compiler_modules() + intent = models.InferenceIntent( + task=models.Generation(chat=True), + model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b"), + engine=models.NativeGlinerEngine(), + placement=models.LocalProcessPlacement(), + access=models.DirectAccess(), + lifecycle=models.ManagedLifecycle(), + ) + + with pytest.raises(compiler.CompilationError) as exc_info: + compiler.compile_intent(intent, source_revision="3f68c145") + + assert exc_info.value.diagnostic.code == "unsupported-task-engine" + assert exc_info.value.diagnostic.details == { + "engine": "native-gliner", + "task": "generation", + } + + +def test_transport_rejects_unknown_fields_and_attach_variants() -> None: + """The initial union is closed and has no attach or external-endpoint path.""" + models, _compiler = load_compiler_modules() + payload = { + "schema_version": "inference-service.intent/v1", + "task": {"kind": "generation", "chat": True}, + "model": {"kind": "hugging-face", "model_id": "openai/gpt-oss-20b"}, + "engine": {"kind": "vllm"}, + "placement": {"kind": "attach", "url": "http://example.invalid/v1"}, + "access": {"kind": "direct"}, + "lifecycle": {"kind": "managed"}, + "unexpected": True, + } + + with pytest.raises(Exception): + models.InferenceIntent.model_validate(payload) + + with pytest.raises(Exception): + models.Generation(chat=False) + + +def test_plan_digest_detects_transport_mutation() -> None: + """Loading a modified plan fails before any runtime effect can occur.""" + models, compiler = load_compiler_modules() + intent = models.InferenceIntent( + task=models.Generation(chat=True), + model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b"), + engine=models.VllmEngine(), + placement=models.LocalProcessPlacement(), + access=models.DirectAccess(), + lifecycle=models.ManagedLifecycle(), + ) + plan = compiler.compile_intent(intent, source_revision="3f68c145") + payload = json.loads(plan.model_dump_json()) + payload["endpoint"]["port"] = 9000 + + with pytest.raises(compiler.PlanIntegrityError, match="plan digest mismatch"): + compiler.load_plan(json.dumps(payload)) + + +def test_compile_command_writes_the_versioned_plan(tmp_path: Path) -> None: + """The CLI is a thin JSON transport over the same pure compiler.""" + cli = load_cli_module() + intent_path = tmp_path / "intent.json" + plan_path = tmp_path / "plan.json" + intent_path.write_text( + json.dumps( + { + "schema_version": "inference-service.intent/v1", + "task": {"kind": "generation", "chat": True}, + "model": {"kind": "hugging-face", "model_id": "openai/gpt-oss-20b"}, + "engine": {"kind": "vllm"}, + "placement": {"kind": "local-process", "host": "127.0.0.1", "port": 8000}, + "access": {"kind": "direct"}, + "lifecycle": {"kind": "managed"}, + } + ), + encoding="utf-8", + ) + + with mock.patch.object(sys, "argv", ["inference-service"]): + with pytest.raises(SystemExit) as exc_info: + cli.app( + [ + "compile", + "--intent", + str(intent_path), + "--source-revision", + "3f68c145", + "--output", + str(plan_path), + ] + ) + + assert exc_info.value.code == 0 + payload = json.loads(plan_path.read_text(encoding="utf-8")) + assert payload["schema_version"] == "inference-service.run-plan/v1" + assert payload["source_revision"] == "3f68c145" + assert payload["plan_digest"] + + +def test_probe_records_generation_capabilities() -> None: + """A live probe records models and observed task capabilities in a v1 receipt.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + assert callable(getattr(runtime, "probe_endpoint", None)) + plan = build_generation_plan(models, compiler) + + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path == "/v1/models": + return httpx.Response(200, json={"data": [{"id": "openai/gpt-oss-20b"}]}) + if request.url.path == "/v1/chat/completions": + return httpx.Response(200, json={"choices": [{"message": {"content": "ready"}}]}) + return httpx.Response(404) + + with httpx.Client(transport=httpx.MockTransport(respond)) as client: + receipt = runtime.probe_endpoint(plan, client=client) + + assert receipt.schema_version == "inference-service.capability-probe-receipt/v1" + assert receipt.plan_digest == plan.plan_digest + assert receipt.models == ("openai/gpt-oss-20b",) + assert receipt.observed_capabilities == ("chat-completions",) + assert receipt.passed is True + + +def test_probe_uses_the_resolved_bearer_secret() -> None: + """Secured readiness and task probes authenticate without serializing the value.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + plan = build_generation_plan(models, compiler, api_key_env="LOCAL_VLLM_API_KEY") + + def respond(request: httpx.Request) -> httpx.Response: + assert request.headers["authorization"] == "Bearer test-secret" + if request.url.path == "/v1/models": + return httpx.Response(200, json={"data": [{"id": "openai/gpt-oss-20b"}]}) + if request.url.path == "/v1/chat/completions": + return httpx.Response(200, json={"choices": [{"message": {"content": "ready"}}]}) + return httpx.Response(404) + + with httpx.Client(transport=httpx.MockTransport(respond)) as client: + receipt = runtime.probe_endpoint( + plan, + client=client, + secret_values={"LOCAL_VLLM_API_KEY": "test-secret"}, + ) + + assert receipt.passed is True + assert "test-secret" not in receipt.model_dump_json() + + +def test_probe_rejects_the_wrong_served_model() -> None: + """Capabilities from another model do not satisfy the compiled contract.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + plan = build_generation_plan(models, compiler) + + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path == "/v1/models": + return httpx.Response(200, json={"data": [{"id": "other/model"}]}) + if request.url.path == "/v1/chat/completions": + return httpx.Response(200, json={"choices": [{"message": {"content": "ready"}}]}) + return httpx.Response(404) + + with httpx.Client(transport=httpx.MockTransport(respond)) as client: + receipt = runtime.probe_endpoint(plan, client=client) + + assert receipt.models == ("other/model",) + assert receipt.observed_capabilities == ("chat-completions",) + assert receipt.passed is False + + +def test_probe_enforces_the_declared_readiness_status() -> None: + """A parseable response with the wrong status does not satisfy readiness.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + plan = build_generation_plan(models, compiler) + + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response(201, json={"data": [{"id": "openai/gpt-oss-20b"}]}) + + with httpx.Client(transport=httpx.MockTransport(respond)) as client: + with pytest.raises(runtime.RuntimeEffectError, match="readiness probe returned status 201, expected 200"): + runtime.probe_endpoint(plan, client=client) + + +def test_launch_local_process_returns_reconnectable_handle(tmp_path: Path) -> None: + """Launching a plan records external process identity and readiness evidence.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + assert callable(getattr(runtime, "launch_plan", None)) + plan = build_generation_plan(models, compiler, api_key_env="LOCAL_VLLM_API_KEY") + probe = models.CapabilityProbeReceipt( + plan_digest=plan.plan_digest, + endpoint=plan.endpoint, + observed_at="2026-08-07T00:00:00+00:00", + models=("openai/gpt-oss-20b",), + observed_capabilities=("chat-completions",), + passed=True, + ) + process = mock.Mock(pid=4242) + + with ( + mock.patch.object(runtime.subprocess, "Popen", return_value=process) as popen, + mock.patch.object(runtime, "probe_endpoint", return_value=probe), + mock.patch.object(runtime, "read_process_start_marker", return_value="100"), + ): + receipt = runtime.launch_plan( + plan, + secret_values={"LOCAL_VLLM_API_KEY": "test-secret"}, + log_directory=tmp_path, + ) + + launched_argv = tuple(popen.call_args.args[0]) + assert launched_argv == plan.command.render_argv() + assert "test-secret" not in launched_argv + assert popen.call_args.kwargs["env"]["VLLM_API_KEY"] == "test-secret" + assert receipt.schema_version == "inference-service.launch-receipt/v1" + assert receipt.plan_digest == plan.plan_digest + assert receipt.handle.kind == "local-process" + assert receipt.handle.external_id == "4242:100" + assert receipt.handle.pid == 4242 + assert receipt.probe == probe + assert Path(receipt.handle.stdout_path).parent == tmp_path + + +def test_launch_rejects_an_unresolved_secret_before_effects(tmp_path: Path) -> None: + """Missing secret references fail before a process or container is started.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + plan = build_generation_plan(models, compiler, api_key_env="LOCAL_VLLM_API_KEY") + + with mock.patch.object(runtime.subprocess, "Popen") as popen: + with pytest.raises(runtime.RuntimeEffectError) as exc_info: + runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) + + popen.assert_not_called() + assert exc_info.value.diagnostic.code == "missing-secret" + assert exc_info.value.diagnostic.known_effects == () + + +def test_launch_rejects_an_in_memory_mutated_plan_before_effects(tmp_path: Path) -> None: + """The Python runtime boundary verifies plans as strictly as the JSON CLI.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + plan = build_generation_plan(models, compiler) + changed = plan.model_copy(update={"expected_model": "other/model"}) + + with mock.patch.object(runtime.subprocess, "Popen") as popen: + with pytest.raises(compiler.PlanIntegrityError, match="plan digest mismatch"): + runtime.launch_plan(changed, secret_values={}, log_directory=tmp_path) + + popen.assert_not_called() + + +def test_launch_docker_returns_container_identity(tmp_path: Path) -> None: + """Docker launch captures the stable container ID instead of the client PID.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + plan = build_generation_plan(models, compiler, docker=True) + probe = models.CapabilityProbeReceipt( + plan_digest=plan.plan_digest, + endpoint=plan.endpoint, + observed_at="2026-08-07T00:00:00+00:00", + models=("openai/gpt-oss-20b",), + observed_capabilities=("chat-completions",), + passed=True, + ) + completed = mock.Mock(returncode=0, stdout="abc123\n", stderr="") + + with ( + mock.patch.object(runtime.subprocess, "run", return_value=completed) as run, + mock.patch.object(runtime, "probe_endpoint", return_value=probe), + ): + receipt = runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) + + assert tuple(run.call_args.args[0]) == plan.command.render_argv() + assert receipt.handle.kind == "docker" + assert receipt.handle.external_id == "abc123" + assert receipt.handle.container_id == "abc123" + + +def test_inspect_and_cancel_local_process_emit_versioned_receipts(tmp_path: Path) -> None: + """A later CLI invocation can inspect and cancel the recorded process identity.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + plan = build_generation_plan(models, compiler) + probe = models.CapabilityProbeReceipt( + plan_digest=plan.plan_digest, + endpoint=plan.endpoint, + observed_at="2026-08-07T00:00:00+00:00", + models=("openai/gpt-oss-20b",), + observed_capabilities=("chat-completions",), + passed=True, + ) + handle = models.LocalProcessHandle( + external_id="4242:100", + pid=4242, + process_group_id=4242, + start_marker="100", + stdout_path=str(tmp_path / "stdout.log"), + stderr_path=str(tmp_path / "stderr.log"), + ) + launch = models.LaunchReceipt( + plan_digest=plan.plan_digest, + launched_at="2026-08-07T00:00:00+00:00", + shutdown_timeout_seconds=10, + handle=handle, + probe=probe, + ) + + with mock.patch.object(runtime, "is_handle_running", return_value=True): + status = runtime.inspect_run(launch) + with ( + mock.patch.object(runtime, "is_handle_running", side_effect=[True, False]), + mock.patch.object(runtime.os, "killpg") as killpg, + ): + cancellation = runtime.cancel_run(launch) + + assert status.schema_version == "inference-service.status-receipt/v1" + assert status.state == "running" + assert cancellation.schema_version == "inference-service.cancellation-receipt/v1" + assert cancellation.outcome == "terminated" + assert cancellation.cleanup_complete is True + killpg.assert_called_once_with(4242, runtime.signal.SIGTERM) + + +def test_process_identity_handles_spaces_and_zombies(tmp_path: Path) -> None: + """Linux process identity parsing handles spaced names and treats zombies as stopped.""" + models, _compiler = load_compiler_modules() + runtime = load_runtime_module() + fields = ["S", *(str(index) for index in range(4, 22)), "98765"] + payload = f"4242 (worker with spaces) {' '.join(fields)}" + assert runtime._parse_process_stat(payload) == ("S", "98765") + handle = models.LocalProcessHandle( + external_id="4242:98765", + pid=4242, + process_group_id=4242, + start_marker="98765", + stdout_path=str(tmp_path / "stdout.log"), + stderr_path=str(tmp_path / "stderr.log"), + ) + with ( + mock.patch.object(runtime, "read_process_start_marker", return_value="98765"), + mock.patch.object(runtime, "read_process_state", return_value="Z"), + mock.patch.object(runtime.os, "kill") as kill, + ): + assert runtime.is_handle_running(handle) is False + kill.assert_not_called() + + +def test_vllm_plan_preserves_lora_model_artifact() -> None: + """LoRA remains a model artifact while vLLM owns its launch spelling.""" + models, compiler = load_compiler_modules() + intent = models.InferenceIntent( + task=models.Generation(chat=True), + model=models.HuggingFaceModel( + model_id="openai/gpt-oss-20b", + adapter=models.LoraAdapter(path="/models/privacy-adapter", name="privacy"), + ), + engine=models.VllmEngine(), + placement=models.LocalProcessPlacement(), + access=models.DirectAccess(), + lifecycle=models.ManagedLifecycle(), + ) + + plan = compiler.compile_intent(intent, source_revision="3f68c145") + + assert plan.command.render_argv()[-3:] == ( + "--enable-lora", + "--lora-modules", + "privacy=/models/privacy-adapter", + ) + + +def test_discover_cached_models_returns_versioned_source_paths(tmp_path: Path) -> None: + """Cached-model discovery survives as typed source-tree output without downloads.""" + models, _compiler = load_compiler_modules() + runtime = load_runtime_module() + assert callable(getattr(runtime, "discover_cached_models", None)) + snapshot = tmp_path / "models--openai--gpt-oss-20b" / "snapshots" / "abc123" + snapshot.mkdir(parents=True) + + result = runtime.discover_cached_models(tmp_path) + + assert result.schema_version == "inference-service.cached-models/v1" + assert result.cache_root == str(tmp_path) + assert result.models == ( + models.CachedModel(repository="openai/gpt-oss-20b", revision="abc123", snapshot_path=str(snapshot)), + ) + + +def test_models_command_writes_versioned_cache_discovery(tmp_path: Path) -> None: + """The CLI retains PR 212's cached-model discovery as typed JSON.""" + cli = load_cli_module() + cache_root = tmp_path / "hub" + snapshot = cache_root / "models--openai--gpt-oss-20b" / "snapshots" / "abc123" + snapshot.mkdir(parents=True) + output = tmp_path / "models.json" + + with pytest.raises(SystemExit) as exc_info: + cli.app(["models", "--cache-root", str(cache_root), "--output", str(output)]) + + assert exc_info.value.code == 0 + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["schema_version"] == "inference-service.cached-models/v1" + assert payload["models"][0]["repository"] == "openai/gpt-oss-20b" + + +def test_native_gliner_cutover_has_no_legacy_entrypoint() -> None: + """The characterized server moves into the compiler tree without a wrapper.""" + assert NATIVE_GLINER_PATH.is_file() + assert NATIVE_GLINER_PATH.stat().st_mode & stat.S_IXUSR + assert not REMOVED_GLINER_PATH.exists() + + +def test_native_gliner_plan_preserves_batch_environment() -> None: + """Compiler plans retain the characterized request-coalescing controls.""" + models, compiler = load_compiler_modules() + intent = models.InferenceIntent( + task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), + model=models.HuggingFaceModel(model_id="nvidia/gliner-pii"), + engine=models.NativeGlinerEngine( + device="cuda", + batch_mode=True, + max_batch_requests=64, + batch_wait_ms=10, + ), + placement=models.LocalProcessPlacement(port=9000), + access=models.DirectAccess(), + lifecycle=models.ManagedLifecycle(), + ) + + plan = compiler.compile_intent(intent, source_revision="3f68c145") + + assert {item.name: item.value for item in plan.command.environment} == { + "DEVICE": "cuda", + "GLINER_BATCH_MODE": "true", + "GLINER_MAX_BATCH_REQUESTS": "64", + "GLINER_BATCH_WAIT_MS": "10.0", + } + + +def test_failed_readiness_cleans_up_the_launched_process(tmp_path: Path) -> None: + """A failed readiness probe reports and cleans every known launch effect.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + plan = build_generation_plan(models, compiler) + process = mock.Mock(pid=4242) + failure = runtime.RuntimeEffectError(models.RuntimeDiagnostic(code="probe-failed", message="not ready")) + + with ( + mock.patch.object(runtime.subprocess, "Popen", return_value=process), + mock.patch.object(runtime, "read_process_start_marker", return_value="100"), + mock.patch.object(runtime, "wait_for_readiness", side_effect=failure), + mock.patch.object(runtime, "is_handle_running", side_effect=[True, False]), + mock.patch.object(runtime.os, "killpg") as killpg, + ): + with pytest.raises(runtime.RuntimeEffectError) as exc_info: + runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) + + assert exc_info.value.diagnostic.known_effects == ("4242:100",) + assert exc_info.value.diagnostic.cleanup_complete is True + killpg.assert_called_once_with(4242, runtime.signal.SIGTERM) diff --git a/tests/tools/test_native_gliner.py b/tests/tools/test_native_gliner.py new file mode 100644 index 00000000..9e1f1dc5 --- /dev/null +++ b/tests/tools/test_native_gliner.py @@ -0,0 +1,276 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Behavior tests for the standalone GLiNER server without local runtimes.""" + +from __future__ import annotations + +import asyncio +import importlib.util +import inspect +import json +import stat +import sys +from pathlib import Path +from types import ModuleType + +import pytest + +SCRIPT_PATH = Path(__file__).parents[2] / "tools" / "inference_service_compiler" / "native_gliner.py" + + +class FakeHTTPException(Exception): + """Small typed stand-in for FastAPI's HTTP exception.""" + + def __init__(self, *, status_code: int, detail: str) -> None: + super().__init__(detail) + self.status_code = status_code + self.detail = detail + + +def fake_module(name: str, **attributes: object) -> ModuleType: + """Build a dynamic dependency module behind one explicit boundary.""" + module = ModuleType(name) + vars(module).update(attributes) + return module + + +def load_server(monkeypatch: pytest.MonkeyPatch) -> ModuleType: + """Load the server after replacing every heavyweight or web dependency.""" + cyclopts = fake_module( + "cyclopts", + App=type("App", (), {"__init__": lambda self, **_kwargs: None, "default": lambda self, function: function}), + Parameter=object, + ) + monkeypatch.setitem(sys.modules, "cyclopts", cyclopts) + fastapi = fake_module( + "fastapi", + FastAPI=type( + "FastAPI", + (), + { + "__init__": lambda self, **_kwargs: None, + "get": lambda self, *_args, **_kwargs: lambda function: function, + "post": lambda self, *_args, **_kwargs: lambda function: function, + }, + ), + HTTPException=FakeHTTPException, + Request=object, + ) + monkeypatch.setitem(sys.modules, "fastapi", fastapi) + structlog = fake_module( + "structlog", + get_logger=lambda _name: type("Logger", (), {"info": lambda self, *_args, **_kwargs: None})(), + make_filtering_bound_logger=lambda _level: object, + configure=lambda **_kwargs: None, + dev=type("Dev", (), {"ConsoleRenderer": lambda: object()}), + processors=type("Processors", (), {"JSONRenderer": lambda: object()}), + ) + monkeypatch.setitem(sys.modules, "structlog", structlog) + monkeypatch.setitem(sys.modules, "uvicorn", ModuleType("uvicorn")) + spec = importlib.util.spec_from_file_location("native_gliner_under_test", SCRIPT_PATH) + if spec is None or spec.loader is None: + raise RuntimeError("could not create the server module specification") + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, module) + spec.loader.exec_module(module) + return module + + +def test_default_nvidia_gliner_selection(monkeypatch: pytest.MonkeyPatch) -> None: + """The default family remains NVIDIA's original PII checkpoint.""" + server = load_server(monkeypatch) + config = server.ServerConfig() + assert config.model is server.ModelFamily.NVIDIA_GLINER + assert config.resolved_checkpoint == "nvidia/gliner-pii" + + +def test_gliner2_selection_and_checkpoint_override(monkeypatch: pytest.MonkeyPatch) -> None: + """GLiNER2 gets its PII default and either family accepts an override.""" + server = load_server(monkeypatch) + gliner2 = server.ServerConfig(model=server.ModelFamily.GLINER2) + overridden = server.ServerConfig(checkpoint="organization/custom-pii") + assert gliner2.resolved_checkpoint == "fastino/gliner2-privacy-filter-PII-multi" + assert overridden.resolved_checkpoint == "organization/custom-pii" + + +def test_invalid_model_value_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + """The typed CLI model choice rejects values outside the closed union.""" + server = load_server(monkeypatch) + with pytest.raises(ValueError, match="not-a-model"): + server.ModelFamily("not-a-model") + + +def test_load_runtime_selects_each_local_api_and_revision(monkeypatch: pytest.MonkeyPatch) -> None: + """Runtime loading dispatches by family and resolves immutable revisions.""" + server = load_server(monkeypatch) + calls: list[tuple[str, str, str, str | None]] = [] + + class NvidiaModel: + @classmethod + def from_pretrained(cls, checkpoint: str, map_location: str, revision: str | None = None) -> object: + calls.append(("nvidia", checkpoint, map_location, revision)) + return object() + + class Gliner2Model: + @classmethod + def from_pretrained(cls, checkpoint: str, map_location: str) -> object: + calls.append(("gliner2", checkpoint, map_location, None)) + return object() + + modules = { + "gliner": type("GlinerModule", (), {"GLiNER": NvidiaModel}), + "gliner2": type("Gliner2Module", (), {"GLiNER2": Gliner2Model}), + "huggingface_hub": type( + "HubModule", + (), + {"snapshot_download": staticmethod(lambda *, repo_id, revision: f"/cache/{repo_id}/{revision}")}, + ), + } + monkeypatch.setattr(server.importlib, "import_module", modules.__getitem__) + nvidia = server.load_runtime(server.ServerConfig(revision="nvidia-revision"), "cpu") + gliner2 = server.load_runtime( + server.ServerConfig( + model=server.ModelFamily.GLINER2, + checkpoint="fastino/custom", + revision="fastino-revision", + ), + "cuda", + ) + assert isinstance(nvidia, server.NvidiaGlinerRuntime) + assert isinstance(gliner2, server.Gliner2Runtime) + assert calls == [ + ("nvidia", "nvidia/gliner-pii", "cpu", "nvidia-revision"), + ("gliner2", "/cache/fastino/custom/fastino-revision", "cuda", None), + ] + + +def test_normalizes_nvidia_spans_and_confidence(monkeypatch: pytest.MonkeyPatch) -> None: + """Original GLiNER dictionaries preserve their span and score values.""" + server = load_server(monkeypatch) + entities = server.normalize_nvidia_output( + [[{"text": "Ada", "label": "person", "start": 3, "end": 6, "score": 0.91}]] + ) + assert entities == [[server.Entity("Ada", "person", 3, 6, 0.91)]] + + +def test_normalizes_gliner2_spans_and_confidence(monkeypatch: pytest.MonkeyPatch) -> None: + """GLiNER2 confidence and spans convert to Anonymizer's flat entity value.""" + server = load_server(monkeypatch) + entities = server.normalize_gliner2_output( + [{"entities": {"email": [{"text": "a@example.com", "start": 5, "end": 18, "confidence": 0.88}]}}] + ) + assert entities == [[server.Entity("a@example.com", "email", 5, 18, 0.88)]] + + +@pytest.mark.parametrize( + ("body", "message"), + [ + ({"labels": [42]}, "labels must be a list of strings"), + ({"flat_ner": "false"}, "flat_ner must be a boolean"), + ({"batch_size": 1.5}, "batch_size must be an integer"), + ({"batch_size": 0}, "batch_size must be >= 1"), + ({"threshold": "0.3"}, "threshold must be a number"), + ({"threshold": 1.1}, "threshold must be between 0 and 1"), + ], +) +def test_request_params_reject_implicit_coercions( + monkeypatch: pytest.MonkeyPatch, body: dict[str, object], message: str +) -> None: + """The functional request boundary rejects ambiguous JSON values.""" + server = load_server(monkeypatch) + with pytest.raises(server.RequestValidationError, match=message): + server.parse_detect_params(body) + + +def test_server_config_rejects_invalid_port_before_startup(monkeypatch: pytest.MonkeyPatch) -> None: + """Invalid ports fail before uvicorn can trigger model initialization.""" + server = load_server(monkeypatch) + with pytest.raises(ValueError, match="port must be between 1 and 65535"): + server.ServerConfig(port=70000) + + +def test_cli_parameters_are_named_options(monkeypatch: pytest.MonkeyPatch) -> None: + """Model and checkpoint selection remain discoverable named options.""" + server = load_server(monkeypatch) + parameters = inspect.signature(server.main).parameters + assert parameters["model"].kind is inspect.Parameter.KEYWORD_ONLY + assert parameters["checkpoint"].kind is inspect.Parameter.KEYWORD_ONLY + assert parameters["log_format"].kind is inspect.Parameter.KEYWORD_ONLY + + +def test_bad_cli_input_uses_craft_exit_code( + monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """CLI validation exits 125 before starting the model server.""" + server = load_server(monkeypatch) + with pytest.raises(SystemExit) as exc_info: + server.main(port=70000) + assert exc_info.value.code == 125 + assert capsys.readouterr().err == "error: port must be between 1 and 65535\n" + + +def test_script_is_directly_executable() -> None: + """The uv shebang and executable mode form a usable entry point.""" + assert SCRIPT_PATH.stat().st_mode & stat.S_IXUSR + + +def test_fastapi_app_alias_is_preserved(monkeypatch: pytest.MonkeyPatch) -> None: + """Existing uvicorn module imports continue to resolve `app`.""" + server = load_server(monkeypatch) + assert server.app is server.api + + +def test_chat_completion_uses_anonymizer_json_string_contract(monkeypatch: pytest.MonkeyPatch) -> None: + """The OpenAI response embeds flat entities in `message.content` JSON.""" + server = load_server(monkeypatch) + + class Detector: + async def detect(self, _text: str, _params: object) -> list[object]: + return [server.Entity("Ada", "person", 0, 3, 0.9)] + + class Request: + async def json(self) -> dict[str, object]: + return {"messages": [{"role": "user", "content": "Ada"}], "labels": ["person"]} + + vars(server)["detector"] = Detector() + response = asyncio.run(server.chat_completions(Request())) + content = response["choices"][0]["message"]["content"] + assert json.loads(content) == {"entities": [{"text": "Ada", "label": "person", "start": 0, "end": 3, "score": 0.9}]} + + +def test_chat_completion_returns_422_for_invalid_params(monkeypatch: pytest.MonkeyPatch) -> None: + """Malformed client values are reported as validation errors, not 500s.""" + server = load_server(monkeypatch) + + class Detector: + async def detect(self, _text: str, _params: object) -> list[object]: + raise AssertionError("invalid requests must not reach inference") + + class Request: + async def json(self) -> dict[str, object]: + return {"messages": [], "labels": [42]} + + vars(server)["detector"] = Detector() + with pytest.raises(FakeHTTPException) as exc_info: + asyncio.run(server.chat_completions(Request())) + assert exc_info.value.status_code == 422 + assert exc_info.value.detail == "labels must be a list of strings" + + +def test_chat_completion_returns_422_for_invalid_messages(monkeypatch: pytest.MonkeyPatch) -> None: + """The message boundary rejects values outside the OpenAI list shape.""" + server = load_server(monkeypatch) + + class Detector: + async def detect(self, _text: str, _params: object) -> list[object]: + raise AssertionError("invalid requests must not reach inference") + + class Request: + async def json(self) -> dict[str, object]: + return {"messages": "bad", "labels": []} + + vars(server)["detector"] = Detector() + with pytest.raises(FakeHTTPException) as exc_info: + asyncio.run(server.chat_completions(Request())) + assert exc_info.value.status_code == 422 + assert exc_info.value.detail == "messages must be a list" diff --git a/tests/tools/test_vllm_debug.py b/tests/tools/test_vllm_debug.py deleted file mode 100644 index 5ab66f25..00000000 --- a/tests/tools/test_vllm_debug.py +++ /dev/null @@ -1,184 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Behavior tests for the local vLLM debug helper.""" - -from __future__ import annotations - -import importlib.util -import sys -from pathlib import Path -from types import ModuleType -from typing import Any - -REPO_ROOT = Path(__file__).resolve().parents[2] -TOOL_PATH = REPO_ROOT / "tools" / "vllm_debug.py" - - -def load_tool() -> ModuleType: - spec = importlib.util.spec_from_file_location("vllm_debug_tool", TOOL_PATH) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -def test_build_serve_command_includes_lora_and_gpu_options() -> None: - tool = load_tool() - - command = tool.build_serve_command( - tool.ServeRequest( - model="HuggingFaceTB/SmolLM3-3B", - host="127.0.0.1", - port=8000, - served_model_name="anonymizer-local", - api_key="test-token", - adapter=Path("/models/adapter"), - adapter_name="anonymizer", - tensor_parallel_size=2, - gpu_memory_utilization=0.8, - max_model_len=4096, - eager=True, - ) - ) - - assert command == [ - sys.executable, - "-m", - "vllm.entrypoints.openai.api_server", - "--model", - "HuggingFaceTB/SmolLM3-3B", - "--host", - "127.0.0.1", - "--port", - "8000", - "--served-model-name", - "anonymizer-local", - "--api-key", - "test-token", - "--enable-lora", - "--lora-modules", - "anonymizer=/models/adapter", - "--tensor-parallel-size", - "2", - "--gpu-memory-utilization", - "0.8", - "--max-model-len", - "4096", - "--enforce-eager", - ] - - -def test_build_serve_command_can_use_a_separate_vllm_interpreter() -> None: - tool = load_tool() - - command = tool.build_serve_command( - tool.ServeRequest( - model="local-model", - python_executable=Path("/opt/vllm/bin/python"), - ) - ) - - assert command[:5] == [ - "/opt/vllm/bin/python", - "-m", - "vllm.entrypoints.openai.api_server", - "--model", - "local-model", - ] - - -def test_render_serve_command_redacts_api_key() -> None: - tool = load_tool() - - rendered = tool.render_serve_command( - tool.build_serve_command(tool.ServeRequest(model="local-model", api_key="test-token")) - ) - - assert "test-token" not in rendered - assert "--api-key ''" in rendered - - -def test_resolve_api_key_reads_a_named_environment_variable(monkeypatch: Any) -> None: - tool = load_tool() - monkeypatch.setenv("LOCAL_VLLM_API_KEY", "test-token") - - assert tool.resolve_api_key(None, "LOCAL_VLLM_API_KEY") == "test-token" - - -def test_discover_cached_models_returns_snapshot_paths(tmp_path: Path) -> None: - tool = load_tool() - snapshot = tmp_path / "models--HuggingFaceTB--SmolLM3-3B" / "snapshots" / "abc123" - snapshot.mkdir(parents=True) - (snapshot / "config.json").write_text("{}", encoding="utf-8") - - models = tool.discover_cached_models(tmp_path) - - assert models == [ - tool.CachedModel( - repository="HuggingFaceTB/SmolLM3-3B", - snapshot_path=snapshot, - ) - ] - - -def test_fetch_models_uses_openai_models_endpoint(monkeypatch: Any) -> None: - tool = load_tool() - calls: list[str] = [] - - class Response: - def raise_for_status(self) -> None: - return None - - def json(self) -> dict[str, object]: - return {"data": [{"id": "local-model"}]} - - def fake_get(url: str, *, timeout: float) -> Response: - calls.append(url) - assert timeout == 10.0 - return Response() - - monkeypatch.setattr(tool.httpx, "get", fake_get) - - assert tool.fetch_models("http://127.0.0.1:8000/v1", timeout_seconds=10.0) == ["local-model"] - assert calls == ["http://127.0.0.1:8000/v1/models"] - - -def test_call_chat_sends_prompt_and_returns_content(monkeypatch: Any) -> None: - tool = load_tool() - request_body: dict[str, object] = {} - - class Response: - def raise_for_status(self) -> None: - return None - - def json(self) -> dict[str, object]: - return { - "choices": [{"message": {"content": "hello"}}], - "usage": {"prompt_tokens": 3, "completion_tokens": 1}, - } - - def fake_post(url: str, *, json: dict[str, object], timeout: float, headers: dict[str, str]) -> Response: - request_body.update(json) - assert url == "http://127.0.0.1:8000/v1/chat/completions" - assert timeout == 15.0 - assert headers == {} - return Response() - - monkeypatch.setattr(tool.httpx, "post", fake_post) - - result = tool.call_chat( - endpoint="http://127.0.0.1:8000/v1", - model="local-model", - prompt="Say hello", - timeout_seconds=15.0, - api_key=None, - ) - - assert request_body == { - "model": "local-model", - "messages": [{"role": "user", "content": "Say hello"}], - } - assert result.content == "hello" - assert result.usage == {"prompt_tokens": 3, "completion_tokens": 1} diff --git a/tools/inference_service.py b/tools/inference_service.py new file mode 100755 index 00000000..99468eee --- /dev/null +++ b/tools/inference_service.py @@ -0,0 +1,20 @@ +#!/usr/bin/env -S uv run --script +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "cyclopts>=3", +# "httpx>=0.27", +# "pydantic>=2.9,<3", +# "structlog>=24.4", +# ] +# /// +"""Compile and manage local inference services from typed intent.""" + +from __future__ import annotations + +from inference_service_compiler.cli import app + +if __name__ == "__main__": + app() diff --git a/tools/inference_service_compiler/__init__.py b/tools/inference_service_compiler/__init__.py new file mode 100644 index 00000000..4f61ada0 --- /dev/null +++ b/tools/inference_service_compiler/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Source-tree inference service compiler owned by Anonymizer.""" + +from __future__ import annotations diff --git a/tools/inference_service_compiler/cli.py b/tools/inference_service_compiler/cli.py new file mode 100644 index 00000000..257aca49 --- /dev/null +++ b/tools/inference_service_compiler/cli.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Command-line transport for the inference service compiler.""" + +from __future__ import annotations + +import functools +import os +import sys +from collections.abc import Callable +from pathlib import Path +from typing import ParamSpec, TypeVar + +import cyclopts +from pydantic import BaseModel, ValidationError + +from inference_service_compiler.compiler import CompilationError, compile_intent, load_plan +from inference_service_compiler.models import InferenceIntent, LaunchReceipt, SecretEnvironmentVariable +from inference_service_compiler.runtime import ( + RuntimeEffectError, + cancel_run, + default_cache_root, + discover_cached_models, + inspect_run, + launch_plan, + probe_endpoint, +) + +app = cyclopts.App(help="Compile and manage local inference services from typed intent.") + +P = ParamSpec("P") +R = TypeVar("R") + + +def command_errors(function: Callable[P, R]) -> Callable[P, R]: + """Render transport and compiler errors with the standard bad-input exit code.""" + + @functools.wraps(function) + def wrapped(*args: P.args, **kwargs: P.kwargs) -> R: + try: + return function(*args, **kwargs) + except (CompilationError, OSError, RuntimeEffectError, ValidationError, ValueError) as exc: + sys.stderr.write(f"error: {exc}\n") + raise SystemExit(125) from exc + + return wrapped + + +@app.command(name="compile") +@command_errors +def compile_plan( + *, + intent: Path, + source_revision: str, + output: Path | None = None, +) -> None: + """Compile a v1 intent JSON document without performing runtime effects.""" + parsed = InferenceIntent.model_validate_json(intent.read_text(encoding="utf-8")) + plan = compile_intent(parsed, source_revision=source_revision) + write_json(plan, output) + + +@app.command +@command_errors +def launch( + *, + plan: Path, + output: Path | None = None, + log_directory: Path = Path(".inference-service-runs"), +) -> None: + """Launch a compiled plan and write its reconnectable handle receipt.""" + parsed = load_plan(plan.read_text(encoding="utf-8")) + required_secrets = { + variable.source_environment_variable + for variable in parsed.command.environment + if isinstance(variable, SecretEnvironmentVariable) + } + secret_values = {name: os.environ[name] for name in required_secrets if name in os.environ} + write_json( + launch_plan(parsed, secret_values=secret_values, log_directory=log_directory), + output, + ) + + +@app.command +@command_errors +def probe(*, plan: Path, output: Path | None = None) -> None: + """Probe the endpoint declared by a managed plan and write capability evidence.""" + parsed = load_plan(plan.read_text(encoding="utf-8")) + source = parsed.readiness.bearer_token_environment_variable + secret_values = {source: os.environ[source]} if source is not None and source in os.environ else {} + write_json(probe_endpoint(parsed, secret_values=secret_values), output) + + +@app.command(name="inspect") +@command_errors +def inspect_command(*, receipt: Path, output: Path | None = None) -> None: + """Inspect the reconnectable identity in a launch receipt.""" + launch_receipt = LaunchReceipt.model_validate_json(receipt.read_text(encoding="utf-8")) + write_json(inspect_run(launch_receipt), output) + + +@app.command +@command_errors +def cancel(*, receipt: Path, output: Path | None = None) -> None: + """Cancel and clean up the managed identity in a launch receipt.""" + launch_receipt = LaunchReceipt.model_validate_json(receipt.read_text(encoding="utf-8")) + write_json(cancel_run(launch_receipt), output) + + +@app.command +@command_errors +def models(*, cache_root: Path | None = None, output: Path | None = None) -> None: + """List existing Hugging Face cache snapshots without downloading models.""" + write_json(discover_cached_models(cache_root or default_cache_root()), output) + + +def write_json(value: BaseModel, output: Path | None) -> None: + """Write one versioned transport value to a file or standard output.""" + rendered = value.model_dump_json(indent=2) + "\n" + if output is None: + sys.stdout.write(rendered) + else: + output.write_text(rendered, encoding="utf-8") diff --git a/tools/inference_service_compiler/compiler.py b/tools/inference_service_compiler/compiler.py new file mode 100644 index 00000000..9856c9b3 --- /dev/null +++ b/tools/inference_service_compiler/compiler.py @@ -0,0 +1,330 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pure compilation from inference intent to an immutable run plan.""" + +from __future__ import annotations + +import hashlib +import hmac +import json + +from pydantic import BaseModel + +from inference_service_compiler.models import ( + Capability, + CommandArgument, + CommandSpec, + CompatibilityEvidence, + DockerPlacement, + DockerRuntime, + EndpointContract, + EntityDetection, + EnvironmentVariable, + FrozenModel, + Generation, + HttpProbe, + InferenceIntent, + LiteralArgument, + LocalProcessPlacement, + LocalProcessRuntime, + NativeGlinerEngine, + RunPlan, + SecretEnvironmentVariable, + VllmEngine, +) + +VLLM_API_KEY_ENV = "VLLM_API_KEY" + + +class CompilerDiagnostic(FrozenModel): + """Serializable reason that semantic intent cannot be compiled.""" + + code: str + message: str + details: dict[str, str] + + +class CompilationError(ValueError): + """Intent failed a closed compiler compatibility rule.""" + + def __init__(self, diagnostic: CompilerDiagnostic) -> None: + super().__init__(diagnostic.message) + self.diagnostic = diagnostic + + +class PlanIntegrityError(ValueError): + """A serialized plan does not match its declared digest.""" + + +def compile_intent(intent: InferenceIntent, *, source_revision: str) -> RunPlan: + """Compile semantic intent without starting, probing, or allocating anything.""" + if not source_revision: + raise ValueError("source_revision must not be empty") + required = intent.task.required_capabilities() + command, runtime, declared, evidence = _compile_service(intent) + placement = intent.placement + endpoint = EndpointContract(host=placement.host, port=placement.port) + plan = RunPlan( + plan_digest="", + intent_digest=digest_model(intent), + intent=intent, + command=command, + runtime=runtime, + endpoint=endpoint, + readiness=HttpProbe( + host=placement.host, + port=placement.port, + path="/v1/models", + timeout_seconds=intent.lifecycle.startup_timeout_seconds, + bearer_token_environment_variable=( + intent.engine.api_key_env if isinstance(intent.engine, VllmEngine) else None + ), + ), + expected_model=intent.expected_model, + required_capabilities=required, + declared_capabilities=declared, + compatibility_evidence=evidence, + source_revision=source_revision, + ) + return plan.model_copy(update={"plan_digest": digest_plan(plan)}) + + +def digest_model(model: BaseModel) -> str: + """Return a stable SHA-256 digest of one typed transport value.""" + payload = model.model_dump(mode="json") + return hashlib.sha256(_canonical_json(payload)).hexdigest() + + +def digest_plan(plan: RunPlan) -> str: + """Return the stable digest of a plan excluding its digest field.""" + payload = plan.model_dump(mode="json", exclude={"plan_digest"}) + return hashlib.sha256(_canonical_json(payload)).hexdigest() + + +def load_plan(serialized: str | bytes) -> RunPlan: + """Parse a closed v1 plan and reject transport mutation.""" + plan = RunPlan.model_validate_json(serialized) + verify_plan(plan) + return plan + + +def verify_plan(plan: RunPlan) -> None: + """Reject a plan whose declared digest does not match its contents.""" + expected = digest_plan(plan) + if not hmac.compare_digest(plan.plan_digest, expected): + raise PlanIntegrityError(f"plan digest mismatch: declared {plan.plan_digest!r}, computed {expected!r}") + + +def _canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() + + +def _compile_service( + intent: InferenceIntent, +) -> tuple[ + CommandSpec, + LocalProcessRuntime | DockerRuntime, + tuple[Capability, ...], + tuple[CompatibilityEvidence, ...], +]: + match intent.engine: + case NativeGlinerEngine() as engine: + if not isinstance(intent.task, EntityDetection): + _raise_unsupported_task_engine(intent.task.kind, engine.kind) + if not isinstance(intent.placement, LocalProcessPlacement): + raise CompilationError( + CompilerDiagnostic( + code="unsupported-engine-placement", + message="native GLiNER is characterized only as a local process", + details={"engine": engine.kind, "placement": intent.placement.kind}, + ) + ) + return _compile_native_gliner(intent, engine) + case VllmEngine() as engine: + return _compile_vllm(intent, engine) + + +def _compile_native_gliner( + intent: InferenceIntent, + engine: NativeGlinerEngine, +) -> tuple[CommandSpec, LocalProcessRuntime, tuple[Capability, ...], tuple[CompatibilityEvidence, ...]]: + placement = intent.placement + if not isinstance(placement, LocalProcessPlacement): + raise TypeError(f"expected LocalProcessPlacement, got {type(placement)!r}") + argv = _literal_arguments( + "uv", + "run", + "--script", + "tools/inference_service_compiler/native_gliner.py", + "--host", + placement.host, + "--port", + str(placement.port), + "--model", + engine.family, + "--checkpoint", + intent.model.model_id, + ) + if engine.log_format != "plain": + argv += _literal_arguments("--log-format", engine.log_format) + if intent.model.revision is not None: + argv += _literal_arguments("--revision", intent.model.revision) + return ( + CommandSpec(argv=argv, environment=_native_environment(engine)), + LocalProcessRuntime(), + intent.task.required_capabilities(), + ( + CompatibilityEvidence( + rule="native-gliner-entity-detection-v1", + outcome="characterized", + detail="Anonymizer chat-completion adapter preserves dynamic labels, offsets, and scores", + ), + ), + ) + + +def _native_environment(engine: NativeGlinerEngine) -> tuple[EnvironmentVariable, ...]: + return ( + EnvironmentVariable(name="DEVICE", value=engine.device), + EnvironmentVariable(name="GLINER_BATCH_MODE", value=str(engine.batch_mode).lower()), + EnvironmentVariable(name="GLINER_MAX_BATCH_REQUESTS", value=str(engine.max_batch_requests)), + EnvironmentVariable(name="GLINER_BATCH_WAIT_MS", value=str(float(engine.batch_wait_ms))), + ) + + +def _compile_vllm( + intent: InferenceIntent, + engine: VllmEngine, +) -> tuple[ + CommandSpec, + LocalProcessRuntime | DockerRuntime, + tuple[Capability, ...], + tuple[CompatibilityEvidence, ...], +]: + command, runtime = _vllm_command(intent, engine) + declared = ("chat-completions",) + outcome = "characterized" if isinstance(intent.task, Generation) else "runtime-probe-required" + evidence = ( + CompatibilityEvidence( + rule="vllm-openai-compatible-v1", + outcome=outcome, + detail=( + "vLLM exposes chat completions for generation" + if outcome == "characterized" + else "entity-detection capabilities must be established by the run probe" + ), + ), + ) + return command, runtime, declared, evidence + + +def _vllm_command( + intent: InferenceIntent, + engine: VllmEngine, +) -> tuple[CommandSpec, LocalProcessRuntime | DockerRuntime]: + engine_arguments = _vllm_engine_arguments(intent, engine) + match intent.placement: + case LocalProcessPlacement() as placement: + argv = _literal_arguments( + engine.executable, + "serve", + intent.model.model_id, + "--host", + placement.host, + "--port", + str(placement.port), + ) + return CommandSpec( + argv=argv + engine_arguments, environment=_vllm_environment(engine) + ), LocalProcessRuntime() + case DockerPlacement() as placement: + return _docker_vllm_command(intent, engine, placement, engine_arguments) + + +def _docker_vllm_command( + intent: InferenceIntent, + engine: VllmEngine, + placement: DockerPlacement, + engine_arguments: tuple[CommandArgument, ...], +) -> tuple[CommandSpec, DockerRuntime]: + values = [ + placement.runtime, + "run", + "--detach", + "--rm", + "--gpus", + placement.gpus, + "--ipc", + "host", + "--publish", + f"{placement.host}:{placement.port}:8000", + ] + if placement.hugging_face_cache is not None: + values.extend(["--volume", f"{placement.hugging_face_cache}:/root/.cache/huggingface"]) + if engine.api_key_env is not None: + values.extend(["--env", VLLM_API_KEY_ENV]) + values.extend([placement.image, "--model", intent.model.model_id]) + return ( + CommandSpec( + argv=_literal_arguments(*values) + engine_arguments, + environment=_vllm_environment(engine), + ), + DockerRuntime(image=placement.image), + ) + + +def _vllm_engine_arguments(intent: InferenceIntent, engine: VllmEngine) -> tuple[CommandArgument, ...]: + arguments: list[CommandArgument] = [] + if intent.model.revision is not None: + arguments.extend( + _literal_arguments( + "--revision", + intent.model.revision, + "--tokenizer-revision", + intent.model.revision, + ) + ) + if engine.served_model_name is not None: + arguments.extend(_literal_arguments("--served-model-name", engine.served_model_name)) + if engine.tensor_parallel_size is not None: + arguments.extend(_literal_arguments("--tensor-parallel-size", str(engine.tensor_parallel_size))) + if engine.gpu_memory_utilization is not None: + arguments.extend(_literal_arguments("--gpu-memory-utilization", str(engine.gpu_memory_utilization))) + if engine.max_model_len is not None: + arguments.extend(_literal_arguments("--max-model-len", str(engine.max_model_len))) + if engine.eager: + arguments.extend(_literal_arguments("--enforce-eager")) + if intent.model.adapter is not None: + arguments.extend( + _literal_arguments( + "--enable-lora", + "--lora-modules", + f"{intent.model.adapter.name}={intent.model.adapter.path}", + ) + ) + return tuple(arguments) + + +def _vllm_environment(engine: VllmEngine) -> tuple[SecretEnvironmentVariable, ...]: + if engine.api_key_env is None: + return () + return ( + SecretEnvironmentVariable( + name=VLLM_API_KEY_ENV, + source_environment_variable=engine.api_key_env, + ), + ) + + +def _literal_arguments(*values: str) -> tuple[CommandArgument, ...]: + return tuple(LiteralArgument(value=value) for value in values) + + +def _raise_unsupported_task_engine(task: str, engine: str) -> None: + raise CompilationError( + CompilerDiagnostic( + code="unsupported-task-engine", + message=f"engine {engine!r} does not support task {task!r}", + details={"engine": engine, "task": task}, + ) + ) diff --git a/tools/inference_service_compiler/models.py b/tools/inference_service_compiler/models.py new file mode 100644 index 00000000..dab658dd --- /dev/null +++ b/tools/inference_service_compiler/models.py @@ -0,0 +1,395 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Versioned transport and immutable intermediate-representation models.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field + +INTENT_SCHEMA_VERSION = "inference-service.intent/v1" +PLAN_SCHEMA_VERSION = "inference-service.run-plan/v1" +CAPABILITY_PROBE_RECEIPT_SCHEMA_VERSION = "inference-service.capability-probe-receipt/v1" +LAUNCH_RECEIPT_SCHEMA_VERSION = "inference-service.launch-receipt/v1" +STATUS_RECEIPT_SCHEMA_VERSION = "inference-service.status-receipt/v1" +CANCELLATION_RECEIPT_SCHEMA_VERSION = "inference-service.cancellation-receipt/v1" + +Capability = Literal["chat-completions", "dynamic-labels", "offsets", "scores"] + + +class FrozenModel(BaseModel): + """Closed immutable base for compiler values and transport records.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class EntityDetection(FrozenModel): + """Entity detection requirements independent of a serving engine.""" + + kind: Literal["entity-detection"] = "entity-detection" + dynamic_labels: bool + offsets: bool + scores: bool + + def required_capabilities(self) -> tuple[Capability, ...]: + """Return the endpoint capabilities required by this task.""" + capabilities: list[Capability] = [] + if self.dynamic_labels: + capabilities.append("dynamic-labels") + if self.offsets: + capabilities.append("offsets") + if self.scores: + capabilities.append("scores") + return tuple(capabilities) + + +class Generation(FrozenModel): + """Text-generation requirements independent of a serving engine.""" + + kind: Literal["generation"] = "generation" + chat: Literal[True] = True + + def required_capabilities(self) -> tuple[Capability, ...]: + """Return the endpoint capabilities required by this task.""" + return ("chat-completions",) + + +TaskSpec = Annotated[EntityDetection | Generation, Field(discriminator="kind")] + + +class LoraAdapter(FrozenModel): + """A LoRA artifact and the stable name exposed by the serving engine.""" + + path: str = Field(min_length=1) + name: str = Field(min_length=1) + + +class HuggingFaceModel(FrozenModel): + """A Hugging Face model identifier and optional immutable revision.""" + + kind: Literal["hugging-face"] = "hugging-face" + model_id: str = Field(min_length=1) + revision: str | None = Field(default=None, min_length=1) + adapter: LoraAdapter | None = None + + +ModelSpec = Annotated[HuggingFaceModel, Field(discriminator="kind")] + + +class NativeGlinerEngine(FrozenModel): + """The characterized local GLiNER or GLiNER2 Python runtime.""" + + kind: Literal["native-gliner"] = "native-gliner" + family: Literal["nvidia-gliner", "gliner2"] = "nvidia-gliner" + device: str = Field(default="auto", min_length=1) + batch_mode: bool = True + max_batch_requests: int = Field(default=32, ge=1) + batch_wait_ms: float = Field(default=10, ge=0) + log_format: Literal["plain", "json"] = "plain" + + +class VllmEngine(FrozenModel): + """vLLM's OpenAI-compatible server with bounded common options.""" + + kind: Literal["vllm"] = "vllm" + executable: str = Field(default="vllm", min_length=1) + served_model_name: str | None = Field(default=None, min_length=1) + api_key_env: str | None = Field(default=None, min_length=1) + tensor_parallel_size: int | None = Field(default=None, ge=1) + gpu_memory_utilization: float | None = Field(default=None, gt=0, le=1) + max_model_len: int | None = Field(default=None, ge=1) + eager: bool = False + + +EngineSpec = Annotated[NativeGlinerEngine | VllmEngine, Field(discriminator="kind")] + + +class LocalProcessPlacement(FrozenModel): + """A process on the caller's host.""" + + kind: Literal["local-process"] = "local-process" + host: str = Field(default="127.0.0.1", min_length=1) + port: int = Field(default=8000, ge=1, le=65535) + + +class DockerPlacement(FrozenModel): + """A managed local Docker container with direct host access.""" + + kind: Literal["docker"] = "docker" + host: str = Field(default="127.0.0.1", min_length=1) + port: int = Field(default=8000, ge=1, le=65535) + image: str = Field(min_length=1) + runtime: Literal["docker"] = "docker" + gpus: str = Field(default="all", min_length=1) + hugging_face_cache: str | None = Field(default=None, min_length=1) + + +PlacementSpec = Annotated[LocalProcessPlacement | DockerPlacement, Field(discriminator="kind")] + + +class DirectAccess(FrozenModel): + """A direct HTTP endpoint exposed by the managed runtime.""" + + kind: Literal["direct"] = "direct" + + +AccessSpec = Annotated[DirectAccess, Field(discriminator="kind")] + + +class ManagedLifecycle(FrozenModel): + """The compiler owns launch, inspection, cancellation, and cleanup.""" + + kind: Literal["managed"] = "managed" + startup_timeout_seconds: float = Field(default=120, gt=0) + shutdown_timeout_seconds: float = Field(default=30, gt=0) + + +LifecycleSpec = Annotated[ManagedLifecycle, Field(discriminator="kind")] + + +class InferenceIntent(FrozenModel): + """Complete semantic input to pure inference-service compilation.""" + + schema_version: Literal["inference-service.intent/v1"] = INTENT_SCHEMA_VERSION + task: TaskSpec + model: ModelSpec + engine: EngineSpec + placement: PlacementSpec + access: AccessSpec + lifecycle: LifecycleSpec + + @property + def expected_model(self) -> str: + """Return the model ID that the compiled endpoint must serve.""" + if self.model.adapter is not None: + return self.model.adapter.name + if isinstance(self.engine, VllmEngine) and self.engine.served_model_name is not None: + return self.engine.served_model_name + return self.model.model_id + + +class LiteralArgument(FrozenModel): + """One non-secret argv value.""" + + kind: Literal["literal"] = "literal" + value: str + + +CommandArgument = LiteralArgument + + +class EnvironmentVariable(FrozenModel): + """One ordinary non-secret process environment value.""" + + kind: Literal["literal"] = "literal" + name: str = Field(min_length=1) + value: str + + +class SecretEnvironmentVariable(FrozenModel): + """One process environment value resolved from a named secret source.""" + + kind: Literal["secret-reference"] = "secret-reference" + name: str = Field(min_length=1) + source_environment_variable: str = Field(min_length=1) + + +EnvironmentSpec = Annotated[EnvironmentVariable | SecretEnvironmentVariable, Field(discriminator="kind")] + + +class CommandSpec(FrozenModel): + """Complete argv and ordinary environment for one managed service.""" + + argv: tuple[CommandArgument, ...] = Field(min_length=1) + environment: tuple[EnvironmentSpec, ...] = () + working_directory: str = "." + + def render_argv(self) -> tuple[str, ...]: + """Render the complete non-secret process argument vector.""" + return tuple(argument.value for argument in self.argv) + + def render_environment(self, *, resolve_secrets: Mapping[str, str] | None = None) -> dict[str, str]: + """Render environment values with redacted or explicitly supplied secrets.""" + values: dict[str, str] = {} + for variable in self.environment: + match variable: + case EnvironmentVariable(name=name, value=value): + values[name] = value + case SecretEnvironmentVariable(name=name, source_environment_variable=source): + if resolve_secrets is None: + values[name] = f"" + elif source in resolve_secrets: + values[name] = resolve_secrets[source] + else: + raise ValueError(f"secret environment variable {source!r} is not resolved") + return values + + +class EndpointContract(FrozenModel): + """Direct OpenAI-compatible endpoint produced by a run.""" + + scheme: Literal["http"] = "http" + host: str + port: int + base_path: Literal["/v1"] = "/v1" + + @property + def url(self) -> str: + """Return the normalized endpoint URL.""" + return f"{self.scheme}://{self.host}:{self.port}{self.base_path}" + + +class HttpProbe(FrozenModel): + """One bounded readiness or capability-probe request.""" + + scheme: Literal["http"] = "http" + host: str + port: int + path: str + expected_status: int = 200 + timeout_seconds: float = Field(gt=0) + bearer_token_environment_variable: str | None = Field(default=None, min_length=1) + + @property + def url(self) -> str: + """Return the complete probe URL.""" + return f"{self.scheme}://{self.host}:{self.port}{self.path}" + + +class LocalProcessRuntime(FrozenModel): + """Runtime facts needed to launch and stop a local process.""" + + kind: Literal["local-process"] = "local-process" + cleanup: Literal["terminate-process-group"] = "terminate-process-group" + + +class DockerRuntime(FrozenModel): + """Runtime facts needed to launch and remove a local container.""" + + kind: Literal["docker"] = "docker" + image: str + cleanup: Literal["remove-container"] = "remove-container" + + +RuntimeSpec = Annotated[LocalProcessRuntime | DockerRuntime, Field(discriminator="kind")] + + +class CompatibilityEvidence(FrozenModel): + """One compiler rule supporting or qualifying the selected combination.""" + + rule: str + outcome: Literal["characterized", "runtime-probe-required"] + detail: str + + +class RunPlan(FrozenModel): + """Portable, immutable, effect-free instructions for one service run.""" + + schema_version: Literal["inference-service.run-plan/v1"] = PLAN_SCHEMA_VERSION + plan_digest: str + intent_digest: str = Field(min_length=1) + intent: InferenceIntent + command: CommandSpec + runtime: RuntimeSpec + endpoint: EndpointContract + readiness: HttpProbe + expected_model: str = Field(min_length=1) + required_capabilities: tuple[Capability, ...] + declared_capabilities: tuple[Capability, ...] + compatibility_evidence: tuple[CompatibilityEvidence, ...] + dependencies: tuple[str, ...] = () + source_revision: str = Field(min_length=1) + + +class CapabilityProbeReceipt(FrozenModel): + """Runtime evidence for the endpoint capabilities required by a plan.""" + + schema_version: Literal["inference-service.capability-probe-receipt/v1"] = CAPABILITY_PROBE_RECEIPT_SCHEMA_VERSION + plan_digest: str + endpoint: EndpointContract + observed_at: str + models: tuple[str, ...] + observed_capabilities: tuple[Capability, ...] + passed: bool + + +class LocalProcessHandle(FrozenModel): + """Reconnectable identity for a managed local process.""" + + kind: Literal["local-process"] = "local-process" + external_id: str + pid: int = Field(ge=1) + process_group_id: int = Field(ge=1) + start_marker: str | None + stdout_path: str + stderr_path: str + + +class DockerHandle(FrozenModel): + """Reconnectable identity for a managed Docker container.""" + + kind: Literal["docker"] = "docker" + external_id: str + container_id: str + + +HandleRecord = Annotated[LocalProcessHandle | DockerHandle, Field(discriminator="kind")] + + +class LaunchReceipt(FrozenModel): + """Known launch effects, reconnectable identity, and readiness evidence.""" + + schema_version: Literal["inference-service.launch-receipt/v1"] = LAUNCH_RECEIPT_SCHEMA_VERSION + plan_digest: str + launched_at: str + shutdown_timeout_seconds: float = Field(gt=0) + handle: HandleRecord + probe: CapabilityProbeReceipt + + +class StatusReceipt(FrozenModel): + """Observed state for a reconnectable managed handle.""" + + schema_version: Literal["inference-service.status-receipt/v1"] = STATUS_RECEIPT_SCHEMA_VERSION + plan_digest: str + observed_at: str + handle: HandleRecord + state: Literal["running", "stopped"] + + +class CancellationReceipt(FrozenModel): + """Cancellation outcome and cleanup state for a managed handle.""" + + schema_version: Literal["inference-service.cancellation-receipt/v1"] = CANCELLATION_RECEIPT_SCHEMA_VERSION + plan_digest: str + canceled_at: str + handle: HandleRecord + outcome: Literal["terminated", "already-stopped", "forced"] + cleanup_complete: bool + + +class RuntimeDiagnostic(FrozenModel): + """Serializable runtime failure including every known external effect.""" + + code: str + message: str + known_effects: tuple[str, ...] = () + cleanup_complete: bool | None = None + + +class CachedModel(FrozenModel): + """One immutable Hugging Face cache snapshot available to local runtimes.""" + + repository: str + revision: str + snapshot_path: str + + +class CachedModels(FrozenModel): + """Versioned discovery result that performs no model downloads.""" + + schema_version: Literal["inference-service.cached-models/v1"] = "inference-service.cached-models/v1" + cache_root: str + models: tuple[CachedModel, ...] diff --git a/tools/inference_service_compiler/native_gliner.py b/tools/inference_service_compiler/native_gliner.py new file mode 100755 index 00000000..4dbdcf13 --- /dev/null +++ b/tools/inference_service_compiler/native_gliner.py @@ -0,0 +1,660 @@ +#!/usr/bin/env -S uv run --script +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "cyclopts>=3.0", +# "fastapi>=0.115", +# "gliner>=0.2.21", +# "gliner2[local]>=1.3", +# "structlog>=24.4", +# "uvicorn>=0.30", +# ] +# /// +"""Serve local GLiNER PII detection through Anonymizer's OpenAI wire contract. + +This server is launched by ``tools/inference_service.py`` from a compiled run +plan. The default ``nvidia-gliner`` family loads ``nvidia/gliner-pii``; +``gliner2`` loads Fastino's local PII checkpoint. +""" + +from __future__ import annotations + +import asyncio +import importlib +import json +import math +import os +import sys +import time +import uuid +from collections.abc import AsyncIterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import asynccontextmanager +from dataclasses import dataclass +from enum import StrEnum +from typing import Protocol, cast + +import structlog # type: ignore[unresolved-import] +import uvicorn +from cyclopts import App +from fastapi import FastAPI, HTTPException, Request # type: ignore[unresolved-import] + +DEFAULT_HOST = "127.0.0.1" +DEFAULT_PORT = 8001 +DEFAULT_CHUNK_LENGTH = 384 +DEFAULT_OVERLAP = 128 +DEFAULT_FLAT_NER = False +DEFAULT_INFERENCE_BATCH_SIZE = 8 +NVIDIA_GLINER_CHECKPOINT = "nvidia/gliner-pii" +GLINER2_CHECKPOINT = "fastino/gliner2-privacy-filter-PII-multi" +BATCH_MODE = os.getenv("GLINER_BATCH_MODE", "true").lower() not in {"0", "false", "no"} +MAX_BATCH_REQUESTS = int(os.getenv("GLINER_MAX_BATCH_REQUESTS", "32")) +BATCH_WAIT_SECONDS = float(os.getenv("GLINER_BATCH_WAIT_MS", "10")) / 1000 + + +class ModelFamily(StrEnum): + """Supported local model runtime families.""" + + NVIDIA_GLINER = "nvidia-gliner" + GLINER2 = "gliner2" + + +class LogFormat(StrEnum): + """Supported server log renderers.""" + + PLAIN = "plain" + JSON = "json" + + +class RequestValidationError(ValueError): + """A malformed detector request at the pure JSON boundary.""" + + +@dataclass(frozen=True, slots=True) +class ServerConfig: + """Immutable server configuration chosen by the CLI.""" + + host: str = DEFAULT_HOST + port: int = DEFAULT_PORT + model: ModelFamily = ModelFamily.NVIDIA_GLINER + checkpoint: str | None = None + revision: str | None = None + + def __post_init__(self) -> None: + """Reject invalid transport values before model side effects.""" + if not 1 <= self.port <= 65535: + raise ValueError("port must be between 1 and 65535") + + @property + def resolved_checkpoint(self) -> str: + """Return the family default unless the user supplied an override.""" + if self.checkpoint: + return self.checkpoint + match self.model: + case ModelFamily.NVIDIA_GLINER: + return NVIDIA_GLINER_CHECKPOINT + case ModelFamily.GLINER2: + return GLINER2_CHECKPOINT + + +@dataclass(frozen=True, slots=True) +class Entity: + """One normalized entity in Anonymizer's detector response shape.""" + + text: str + label: str + start: int + end: int + score: float + + def as_dict(self) -> dict[str, str | int | float]: + """Serialize the entity in Anonymizer's required flat schema.""" + return {"text": self.text, "label": self.label, "start": self.start, "end": self.end, "score": self.score} + + +@dataclass(frozen=True, slots=True) +class DetectParams: + """Per-request inference settings that determine batching compatibility.""" + + labels: tuple[str, ...] + threshold: float + chunk_length: int + overlap: int + flat_ner: bool + inference_batch_size: int + + +class LocalRuntime(Protocol): + """Narrow common contract for local inference adapters.""" + + def infer(self, chunks: list[str], params: DetectParams) -> list[list[Entity]]: + """Detect normalized entities for each chunk.""" + + +class NvidiaGlinerRuntime: + """Adapter for the original `gliner` local inference API.""" + + def __init__(self, model: object) -> None: + self._model = model + + def infer(self, chunks: list[str], params: DetectParams) -> list[list[Entity]]: + raw = cast( + object, + self._model.inference( # type: ignore[attr-defined] + texts=chunks, + labels=list(params.labels), + threshold=params.threshold, + flat_ner=params.flat_ner, + relations=[], + batch_size=params.inference_batch_size, + ), + ) + return normalize_nvidia_output(raw) + + +class Gliner2Runtime: + """Adapter for GLiNER2's local batch extraction API.""" + + def __init__(self, model: object) -> None: + self._model = model + + def infer(self, chunks: list[str], params: DetectParams) -> list[list[Entity]]: + raw = cast( + object, + self._model.batch_extract_entities( # type: ignore[attr-defined] + chunks, + list(params.labels), + threshold=params.threshold, + include_confidence=True, + include_spans=True, + batch_size=params.inference_batch_size, + ), + ) + return normalize_gliner2_output(raw) + + +def create_text_chunks(text: str, chunk_length: int, overlap: int) -> tuple[list[str], list[int]]: + """Split text into overlapping chunks while retaining their global offsets.""" + chunks: list[str] = [] + offsets: list[int] = [] + start = 0 + while start < len(text): + chunks.append(text[start : start + chunk_length]) + offsets.append(start) + if start + chunk_length >= len(text): + break + start += chunk_length - overlap + return chunks, offsets + + +def normalize_nvidia_output(raw: object) -> list[list[Entity]]: + """Convert original GLiNER dictionaries at the runtime boundary. + + Args: + raw: The `GLiNER.inference` batch result. + + Returns: + One normalized entity list per input chunk. + + Raises: + ValueError: If the runtime did not return GLiNER's documented batch shape. + """ + chunks = require_list(raw, "nvidia-gliner inference batch") + return [ + [normalize_nvidia_entity(entity) for entity in require_list(chunk, "nvidia-gliner chunk")] for chunk in chunks + ] + + +def normalize_gliner2_output(raw: object) -> list[list[Entity]]: + """Convert GLiNER2 entity records, including confidence and character spans. + + Args: + raw: The `GLiNER2.batch_extract_entities` result. + + Returns: + One normalized entity list per input chunk. + + Raises: + ValueError: If the runtime did not return a list of result mappings. + """ + return [ + normalize_gliner2_result(require_mapping(result, "gliner2 result")) + for result in require_list(raw, "gliner2 inference batch") + ] + + +def load_runtime(config: ServerConfig, device: str) -> LocalRuntime: + """Load the selected local runtime at the sole heavyweight side-effect boundary. + + Args: + config: Selected model family and checkpoint. + device: Local Torch device name. + + Returns: + A runtime adapter for inference. + """ + match config.model: + case ModelFamily.NVIDIA_GLINER: + gliner = importlib.import_module("gliner") + model = gliner.GLiNER.from_pretrained( + config.resolved_checkpoint, + map_location=device, + revision=config.revision, + ) + return NvidiaGlinerRuntime(model) + case ModelFamily.GLINER2: + gliner2 = importlib.import_module("gliner2") + checkpoint = config.resolved_checkpoint + if config.revision is not None: + hub = importlib.import_module("huggingface_hub") + checkpoint = hub.snapshot_download(repo_id=checkpoint, revision=config.revision) + model = gliner2.GLiNER2.from_pretrained(checkpoint, map_location=device) + return Gliner2Runtime(model) + + +def resolve_device() -> str: + """Choose an explicit DEVICE override or the best available local accelerator.""" + requested = os.getenv("DEVICE", "auto") + if requested != "auto": + return requested + torch = importlib.import_module("torch") + if torch.backends.mps.is_available(): + return "mps" + if torch.cuda.is_available(): + return "cuda" + return "cpu" + + +def finalize_entities(entities: list[Entity], *, flat_ner: bool) -> list[Entity]: + """Deduplicate overlap artifacts and optionally remove nested spans.""" + candidates = entities if flat_ner else remove_subset_entities(entities) + best: dict[tuple[str, str, int, int], Entity] = {} + for entity in candidates: + key = (entity.label, entity.text.strip().lower(), entity.start, entity.end) + if key not in best or entity.score > best[key].score: + best[key] = entity + return list(best.values()) + + +def remove_subset_entities(entities: list[Entity]) -> list[Entity]: + """Discard an entity wholly contained by a distinct larger entity.""" + return [ + entity + for entity in entities + if not any( + other != entity + and other.start <= entity.start + and other.end >= entity.end + and (other.start < entity.start or other.end > entity.end) + for other in entities + ) + ] + + +def detect_entities_for_texts(runtime: LocalRuntime, texts: list[str], params: DetectParams) -> list[list[Entity]]: + """Run all text chunks through one runtime batch and restore global offsets.""" + if not params.labels: + return [[] for _ in texts] + records = [ + (text_index, offset, chunk) + for text_index, text in enumerate(texts) + if text + for chunk, offset in zip(*create_text_chunks(text, params.chunk_length, params.overlap), strict=True) + ] + output = [[] for _ in texts] + if not records: + return output + inferred = runtime.infer([chunk for _, _, chunk in records], params) + for (text_index, offset, _), entities in zip(records, inferred, strict=True): + output[text_index].extend( + Entity(entity.text, entity.label, entity.start + offset, entity.end + offset, entity.score) + for entity in entities + ) + return [finalize_entities(entities, flat_ner=params.flat_ner) for entities in output] + + +@dataclass(slots=True) +class DetectJob: + """Queued request awaiting a shared inference call.""" + + text: str + params: DetectParams + future: asyncio.Future[list[Entity]] + + +class BatchDetector: + """Coalesce compatible requests while retaining one inference executor.""" + + def __init__(self, runtime: LocalRuntime) -> None: + self._runtime = runtime + self._queue: asyncio.Queue[DetectJob | None] = asyncio.Queue() + self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="gliner-infer") + self._worker_task: asyncio.Task[None] | None = None + + def start(self) -> None: + """Start the request-coalescing worker.""" + self._worker_task = asyncio.create_task(self._worker()) + + async def stop(self) -> None: + """Drain and stop the worker and its dedicated inference executor.""" + if self._worker_task is not None: + await self._queue.put(None) + await self._worker_task + self._executor.shutdown(wait=True) + + async def detect(self, text: str, params: DetectParams) -> list[Entity]: + """Queue one detection request or execute it serially when batching is off.""" + loop = asyncio.get_running_loop() + if not BATCH_MODE: + return ( + await loop.run_in_executor(self._executor, detect_entities_for_texts, self._runtime, [text], params) + )[0] + future: asyncio.Future[list[Entity]] = loop.create_future() + await self._queue.put(DetectJob(text, params, future)) + return await future + + async def _worker(self) -> None: + while first := await self._queue.get(): + jobs = [first] + deadline = asyncio.get_running_loop().time() + BATCH_WAIT_SECONDS + while len(jobs) < MAX_BATCH_REQUESTS: + try: + queued = await asyncio.wait_for( + self._queue.get(), max(0, deadline - asyncio.get_running_loop().time()) + ) + except TimeoutError: + break + if queued is None: + await self._queue.put(None) + break + jobs.append(queued) + await self._dispatch(jobs) + + async def _dispatch(self, jobs: list[DetectJob]) -> None: + groups: dict[DetectParams, list[DetectJob]] = {} + for job in jobs: + groups.setdefault(job.params, []).append(job) + loop = asyncio.get_running_loop() + for params, group in groups.items(): + try: + results = await loop.run_in_executor( + self._executor, detect_entities_for_texts, self._runtime, [job.text for job in group], params + ) + except Exception as exc: + for job in group: + if not job.future.done(): + job.future.set_exception(exc) + else: + for job, entities in zip(group, results, strict=True): + if not job.future.done(): + job.future.set_result(entities) + + +def normalize_nvidia_entity(raw: object) -> Entity: + """Normalize one original GLiNER entity dictionary.""" + mapping = require_mapping(raw, "nvidia-gliner entity") + return Entity( + str(mapping["text"]), + str(mapping["label"]), + coerce_int(mapping["start"], "nvidia-gliner start"), + coerce_int(mapping["end"], "nvidia-gliner end"), + coerce_float(mapping["score"], "nvidia-gliner score"), + ) + + +def normalize_gliner2_result(raw: Mapping[str, object]) -> list[Entity]: + """Normalize one GLiNER2 result mapping keyed by entity label.""" + entities = require_mapping(raw.get("entities"), "gliner2 entities") + return [ + normalize_gliner2_entity(label, item) + for label, values in entities.items() + for item in require_sequence(values, "gliner2 entity values") + ] + + +def normalize_gliner2_entity(label: object, raw: object) -> Entity: + """Normalize GLiNER2's span/confidence record for a named label.""" + entity = require_mapping(raw, "gliner2 entity") + span = entity.get("span") + if isinstance(span, Sequence) and not isinstance(span, str) and len(span) == 2: + start, end = span + else: + start, end = entity["start"], entity["end"] + confidence = entity.get("confidence", entity.get("score")) + if confidence is None: + raise ValueError("gliner2 entity is missing confidence") + return Entity( + str(entity["text"]), + str(label), + coerce_int(start, "gliner2 start"), + coerce_int(end, "gliner2 end"), + coerce_float(confidence, "gliner2 confidence"), + ) + + +def require_mapping(value: object, name: str) -> Mapping[str, object]: + """Validate an untyped runtime mapping at the adapter boundary.""" + if not isinstance(value, Mapping) or any(not isinstance(key, str) for key in value): + raise ValueError(f"unexpected {name} shape") + return cast(Mapping[str, object], value) + + +def require_sequence(value: object, name: str) -> Sequence[object]: + """Validate an untyped runtime sequence at the adapter boundary.""" + if not isinstance(value, Sequence) or isinstance(value, str): + raise ValueError(f"unexpected {name} shape") + return value + + +def require_list(value: object, name: str) -> list[object]: + """Validate a runtime list at the adapter boundary.""" + if not isinstance(value, list): + raise ValueError(f"unexpected {name} shape") + return cast(list[object], value) + + +def coerce_int(value: object, name: str) -> int: + """Convert a runtime numeric value or raise a boundary-specific error.""" + if isinstance(value, bool) or not isinstance(value, str | int | float): + raise ValueError(f"unexpected {name} value") + return int(value) + + +def coerce_float(value: object, name: str) -> float: + """Convert a runtime numeric value or raise a boundary-specific error.""" + if isinstance(value, bool) or not isinstance(value, str | int | float): + raise ValueError(f"unexpected {name} value") + return float(value) + + +def require_request_int(value: object, name: str) -> int: + """Accept only a JSON integer, excluding booleans and lossy coercions.""" + match value: + case bool(): + raise RequestValidationError(f"{name} must be an integer") + case int(): + return value + case _: + raise RequestValidationError(f"{name} must be an integer") + + +def require_request_float(value: object, name: str) -> float: + """Accept a finite JSON number without parsing strings or booleans.""" + match value: + case bool(): + raise RequestValidationError(f"{name} must be a number") + case int() | float(): + number = float(value) + case _: + raise RequestValidationError(f"{name} must be a number") + if not math.isfinite(number): + raise RequestValidationError(f"{name} must be finite") + return number + + +def require_request_bool(value: object, name: str) -> bool: + """Accept a JSON boolean without truthiness coercion.""" + match value: + case bool(): + return value + case _: + raise RequestValidationError(f"{name} must be a boolean") + + +def parse_detect_params(body: Mapping[str, object]) -> DetectParams: + """Parse and validate request options without performing I/O.""" + raw_labels = body.get("labels", []) + match raw_labels: + case list() if all(isinstance(label, str) for label in raw_labels): + labels = tuple(cast(str, label) for label in raw_labels) + case _: + raise RequestValidationError("labels must be a list of strings") + + threshold = require_request_float(body.get("threshold", 0.3), "threshold") + if not 0 <= threshold <= 1: + raise RequestValidationError("threshold must be between 0 and 1") + + chunk_length = require_request_int(body.get("chunk_length", DEFAULT_CHUNK_LENGTH), "chunk_length") + overlap = require_request_int(body.get("overlap", DEFAULT_OVERLAP), "overlap") + flat_ner = require_request_bool(body.get("flat_ner", DEFAULT_FLAT_NER), "flat_ner") + inference_batch_size = require_request_int(body.get("batch_size", DEFAULT_INFERENCE_BATCH_SIZE), "batch_size") + if inference_batch_size < 1: + raise RequestValidationError("batch_size must be >= 1") + validate_chunk_params(chunk_length, overlap) + return DetectParams(labels, threshold, chunk_length, overlap, flat_ner, inference_batch_size) + + +def extract_text(messages: object) -> str: + """Extract text from the final user message's string or multipart content.""" + if not isinstance(messages, list): + raise RequestValidationError("messages must be a list") + if not messages: + return "" + message = require_mapping(messages[-1], "message") + content = message.get("content", "") + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for part in content: + text = require_mapping(part, "message part").get("text", "") + if not isinstance(text, str): + raise RequestValidationError("message part text must be a string") + parts.append(text) + return "".join(parts) + raise RequestValidationError("message content must be a string or list") + + +def validate_chunk_params(chunk_length: int, overlap: int) -> None: + """Reject invalid chunk settings before inference.""" + if chunk_length < 1: + raise RequestValidationError("chunk_length must be >= 1") + if overlap < 0 or overlap >= chunk_length: + raise RequestValidationError("overlap must be >= 0 and less than chunk_length") + + +state: ServerConfig | None = None +runtime: LocalRuntime | None = None +detector: BatchDetector | None = None +log = structlog.get_logger("gliner-server") + + +def configure_logging(log_format: LogFormat) -> None: + """Configure the selected human-readable or structured log renderer.""" + match log_format: + case LogFormat.PLAIN: + renderer = structlog.dev.ConsoleRenderer() + case LogFormat.JSON: + renderer = structlog.processors.JSONRenderer() + structlog.configure(processors=[renderer], wrapper_class=structlog.make_filtering_bound_logger(20)) + + +@asynccontextmanager +async def lifespan(_api: FastAPI) -> AsyncIterator[None]: + """Own the local runtime and its single inference worker for API lifetime.""" + global runtime, detector + if state is None: + raise RuntimeError("server configuration is not initialized") + device = resolve_device() + runtime = await asyncio.to_thread(load_runtime, state, device) + detector = BatchDetector(runtime) + detector.start() + log.info("server_ready", model=state.model, checkpoint=state.resolved_checkpoint, device=device) + try: + yield + finally: + if detector is not None: + await detector.stop() + + +api = FastAPI(lifespan=lifespan) +app = api + + +@api.get("/v1/models") +def list_models() -> dict[str, object]: + """Return the selected local checkpoint in OpenAI's model-list shape.""" + checkpoint = state.resolved_checkpoint if state else NVIDIA_GLINER_CHECKPOINT + return {"object": "list", "data": [{"id": checkpoint, "object": "model"}]} + + +@api.post("/v1/chat/completions") +async def chat_completions(request: Request) -> dict[str, object]: + """Detect requested entity labels and return Anonymizer's JSON-string content.""" + if detector is None: + raise HTTPException(status_code=503, detail="GLiNER model is not loaded") + try: + body = require_mapping(await request.json(), "request") + params = parse_detect_params(body) + text = extract_text(body.get("messages", [])) + except ValueError as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + entities = await detector.detect(text, params) + content = json.dumps({"entities": [entity.as_dict() for entity in entities]}) + return { + "id": f"chatcmpl-{uuid.uuid4().hex[:12]}", + "object": "chat.completion", + "created": int(time.time()), + "model": str(body.get("model", state.resolved_checkpoint if state else NVIDIA_GLINER_CHECKPOINT)), + "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + } + + +cli = App(help="OpenAI-compatible local GLiNER server for Anonymizer.") + + +@cli.default +def main( + *, + host: str = DEFAULT_HOST, + port: int = DEFAULT_PORT, + model: ModelFamily = ModelFamily.NVIDIA_GLINER, + checkpoint: str | None = None, + revision: str | None = None, + log_format: LogFormat = LogFormat.PLAIN, +) -> None: + """Run the server without contacting any remote inference service. + + Args: + host: Bind address; use a private address unless protected by a proxy. + port: TCP listen port. + model: `nvidia-gliner` or `gliner2` local model family. + checkpoint: Optional Hugging Face checkpoint override for that family. + revision: Optional immutable Hugging Face model revision. + log_format: Human-readable `plain` logs or newline-delimited `json`. + """ + global state + try: + state = ServerConfig(host=host, port=port, model=model, checkpoint=checkpoint, revision=revision) + except ValueError as exc: + sys.stderr.write(f"error: {exc}\n") + raise SystemExit(125) from exc + configure_logging(log_format) + uvicorn.run(api, host=host, port=port) + + +if __name__ == "__main__": + cli() diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py new file mode 100644 index 00000000..4e9b54db --- /dev/null +++ b/tools/inference_service_compiler/runtime.py @@ -0,0 +1,472 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Explicit local-process and Docker effects for immutable run plans.""" + +from __future__ import annotations + +import json +import os +import signal +import subprocess +import time +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Literal, cast + +import httpx + +from inference_service_compiler.compiler import verify_plan +from inference_service_compiler.models import ( + CachedModel, + CachedModels, + CancellationReceipt, + Capability, + CapabilityProbeReceipt, + DockerHandle, + DockerRuntime, + EntityDetection, + Generation, + LaunchReceipt, + LocalProcessHandle, + LocalProcessRuntime, + RunPlan, + RuntimeDiagnostic, + SecretEnvironmentVariable, + StatusReceipt, +) + + +class RuntimeEffectError(RuntimeError): + """A launch or probe failed with a serializable partial-effects record.""" + + def __init__(self, diagnostic: RuntimeDiagnostic) -> None: + super().__init__(diagnostic.message) + self.diagnostic = diagnostic + + +def discover_cached_models(cache_root: Path) -> CachedModels: + """List existing Hugging Face snapshots without creating or downloading files.""" + discovered: list[CachedModel] = [] + if cache_root.exists(): + for model_directory in sorted(cache_root.glob("models--*")): + repository = model_directory.name.removeprefix("models--").replace("--", "/") + for snapshot in sorted((model_directory / "snapshots").glob("*")): + if snapshot.is_dir(): + discovered.append( + CachedModel( + repository=repository, + revision=snapshot.name, + snapshot_path=str(snapshot), + ) + ) + return CachedModels(cache_root=str(cache_root), models=tuple(discovered)) + + +def default_cache_root() -> Path: + """Resolve the Hugging Face cache location without creating it.""" + if hub_cache := os.getenv("HF_HUB_CACHE"): + return Path(hub_cache) + if hf_home := os.getenv("HF_HOME"): + return Path(hf_home) / "hub" + return Path.home() / ".cache" / "huggingface" / "hub" + + +def probe_endpoint( + plan: RunPlan, + *, + client: httpx.Client | None = None, + secret_values: Mapping[str, str] | None = None, +) -> CapabilityProbeReceipt: + """Probe one managed endpoint and record only capabilities observed at runtime.""" + verify_plan(plan) + headers = _probe_headers(plan, secret_values or {}) + owns_client = client is None + active_client = client or httpx.Client(timeout=10) + try: + models_response = active_client.get(plan.readiness.url, headers=headers) + if models_response.status_code != plan.readiness.expected_status: + raise ValueError( + f"readiness probe returned status {models_response.status_code}, " + f"expected {plan.readiness.expected_status}" + ) + models = _parse_models(models_response.json()) + observed = _probe_task(plan, active_client, headers) + except (httpx.HTTPError, KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise RuntimeEffectError( + RuntimeDiagnostic(code="probe-failed", message=f"capability probe failed: {exc}") + ) from exc + finally: + if owns_client: + active_client.close() + return CapabilityProbeReceipt( + plan_digest=plan.plan_digest, + endpoint=plan.endpoint, + observed_at=_now(), + models=models, + observed_capabilities=observed, + passed=plan.expected_model in models and set(plan.required_capabilities).issubset(observed), + ) + + +def launch_plan( + plan: RunPlan, + *, + secret_values: dict[str, str], + log_directory: Path, +) -> LaunchReceipt: + """Launch a verified plan and return reconnectable identity plus readiness evidence.""" + verify_plan(plan) + resolved_secrets = _resolve_secrets(plan, secret_values) + argv = plan.command.render_argv() + environment = os.environ.copy() + environment.update(plan.command.render_environment(resolve_secrets=resolved_secrets)) + log_directory.mkdir(parents=True, exist_ok=True) + handle = _launch_handle(plan, argv, environment, log_directory) + probe = _probe_or_cleanup(plan, handle, resolved_secrets) + return LaunchReceipt( + plan_digest=plan.plan_digest, + launched_at=_now(), + shutdown_timeout_seconds=plan.intent.lifecycle.shutdown_timeout_seconds, + handle=handle, + probe=probe, + ) + + +def _launch_handle( + plan: RunPlan, + argv: tuple[str, ...], + environment: dict[str, str], + log_directory: Path, +) -> LocalProcessHandle | DockerHandle: + match plan.runtime: + case LocalProcessRuntime(): + return _launch_process(plan, argv, environment, log_directory) + case DockerRuntime(): + return _launch_docker(argv, environment) + + +def _probe_or_cleanup( + plan: RunPlan, + handle: LocalProcessHandle | DockerHandle, + secret_values: Mapping[str, str], +) -> CapabilityProbeReceipt: + try: + probe = wait_for_readiness(plan, secret_values=secret_values) + except RuntimeEffectError as exc: + cleanup_complete = _cleanup_handle(handle, plan.intent.lifecycle.shutdown_timeout_seconds) + raise RuntimeEffectError( + exc.diagnostic.model_copy( + update={ + "known_effects": (handle.external_id,), + "cleanup_complete": cleanup_complete, + } + ) + ) from exc + if not probe.passed: + cleanup_complete = _cleanup_handle(handle, plan.intent.lifecycle.shutdown_timeout_seconds) + raise RuntimeEffectError( + RuntimeDiagnostic( + code="capability-mismatch", + message="endpoint became ready but did not satisfy required capabilities", + known_effects=(handle.external_id,), + cleanup_complete=cleanup_complete, + ) + ) + return probe + + +def inspect_run(launch: LaunchReceipt) -> StatusReceipt: + """Inspect a reconnectable handle without changing its state.""" + state = "running" if is_handle_running(launch.handle) else "stopped" + return StatusReceipt( + plan_digest=launch.plan_digest, + observed_at=_now(), + handle=launch.handle, + state=state, + ) + + +def cancel_run(launch: LaunchReceipt) -> CancellationReceipt: + """Stop the exact process group or container recorded by a launch receipt.""" + handle = launch.handle + if not is_handle_running(handle): + return CancellationReceipt( + plan_digest=launch.plan_digest, + canceled_at=_now(), + handle=handle, + outcome="already-stopped", + cleanup_complete=True, + ) + outcome, cleanup_complete = _stop_running_handle(handle, launch.shutdown_timeout_seconds) + return CancellationReceipt( + plan_digest=launch.plan_digest, + canceled_at=_now(), + handle=handle, + outcome=outcome, + cleanup_complete=cleanup_complete, + ) + + +def is_handle_running(handle: LocalProcessHandle | DockerHandle) -> bool: + """Check the external identity while guarding against Linux PID reuse.""" + match handle: + case LocalProcessHandle(): + current_marker = read_process_start_marker(handle.pid) + if handle.start_marker is not None and current_marker != handle.start_marker: + return False + if read_process_state(handle.pid) == "Z": + return False + try: + os.kill(handle.pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + case DockerHandle(): + completed = subprocess.run( + ["docker", "inspect", "--format", "{{.State.Running}}", handle.container_id], + capture_output=True, + text=True, + check=False, + ) + return completed.returncode == 0 and completed.stdout.strip().lower() == "true" + + +def _cleanup_handle(handle: LocalProcessHandle | DockerHandle, timeout_seconds: float) -> bool: + if not is_handle_running(handle): + return True + _outcome, cleanup_complete = _stop_running_handle(handle, timeout_seconds) + return cleanup_complete + + +def _stop_running_handle( + handle: LocalProcessHandle | DockerHandle, + timeout_seconds: float, +) -> tuple[Literal["terminated", "forced"], bool]: + match handle: + case LocalProcessHandle(): + try: + os.killpg(handle.process_group_id, signal.SIGTERM) + except ProcessLookupError: + return "terminated", True + deadline = time.monotonic() + timeout_seconds + running = True + while time.monotonic() < deadline: + running = is_handle_running(handle) + if not running: + break + time.sleep(0.1) + outcome: Literal["terminated", "forced"] = "terminated" + if running: + try: + os.killpg(handle.process_group_id, signal.SIGKILL) + except ProcessLookupError: + return "forced", True + running = is_handle_running(handle) + outcome = "forced" + return outcome, not running + case DockerHandle(): + completed = subprocess.run( + ["docker", "stop", "--time", str(int(timeout_seconds)), handle.container_id], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + raise RuntimeEffectError( + RuntimeDiagnostic( + code="docker-cancel-failed", + message=completed.stderr.strip() or "docker stop failed", + known_effects=(handle.external_id,), + cleanup_complete=False, + ) + ) + return "terminated", True + + +def wait_for_readiness( + plan: RunPlan, + *, + secret_values: Mapping[str, str] | None = None, +) -> CapabilityProbeReceipt: + """Poll the declared readiness contract until it passes or times out.""" + deadline = time.monotonic() + plan.readiness.timeout_seconds + last_error: RuntimeEffectError | None = None + while time.monotonic() < deadline: + try: + receipt = probe_endpoint(plan, secret_values=secret_values) + if receipt.passed: + return receipt + last_error = RuntimeEffectError( + RuntimeDiagnostic(code="capability-mismatch", message="required capabilities were not observed") + ) + except RuntimeEffectError as exc: + last_error = exc + time.sleep(min(0.5, max(0, deadline - time.monotonic()))) + message = str(last_error) if last_error is not None else "readiness probe timed out" + raise RuntimeEffectError(RuntimeDiagnostic(code="readiness-timeout", message=message)) + + +def read_process_start_marker(pid: int) -> str | None: + """Read Linux process start ticks to disambiguate PID reuse when available.""" + stat = _read_process_stat(pid) + return stat[1] if stat is not None else None + + +def read_process_state(pid: int) -> str | None: + """Read the Linux process state so zombies count as stopped.""" + stat = _read_process_stat(pid) + return stat[0] if stat is not None else None + + +def _read_process_stat(pid: int) -> tuple[str, str] | None: + stat_path = Path("/proc") / str(pid) / "stat" + try: + payload = stat_path.read_text(encoding="utf-8") + except OSError: + return None + return _parse_process_stat(payload) + + +def _parse_process_stat(payload: str) -> tuple[str, str] | None: + _prefix, separator, suffix = payload.rpartition(")") + fields = suffix.strip().split() if separator else [] + return (fields[0], fields[19]) if len(fields) > 19 else None + + +def _probe_task(plan: RunPlan, client: httpx.Client, headers: Mapping[str, str]) -> tuple[Capability, ...]: + match plan.intent.task: + case Generation(): + response = client.post( + f"{plan.endpoint.url}/chat/completions", + json={ + "model": plan.expected_model, + "messages": [{"role": "user", "content": "Reply with the word ready."}], + "max_tokens": 8, + }, + headers=headers, + ) + response.raise_for_status() + content = response.json()["choices"][0]["message"]["content"] + if not isinstance(content, str): + raise ValueError("chat completion content must be a string") + return ("chat-completions",) + case EntityDetection(): + response = client.post( + f"{plan.endpoint.url}/chat/completions", + json={ + "model": plan.expected_model, + "messages": [{"role": "user", "content": "Ada Lovelace"}], + "labels": ["person"], + "threshold": 0.1, + }, + headers=headers, + ) + response.raise_for_status() + content = response.json()["choices"][0]["message"]["content"] + payload = json.loads(content) + entities = payload["entities"] + if not isinstance(entities, list): + raise ValueError("detector entities must be a list") + observed: list[Capability] = ["dynamic-labels"] + if entities and all(isinstance(entity, dict) and {"start", "end"} <= entity.keys() for entity in entities): + observed.append("offsets") + if entities and all(isinstance(entity, dict) and "score" in entity for entity in entities): + observed.append("scores") + return tuple(observed) + raise TypeError(f"unsupported task type {type(plan.intent.task)!r}") + + +def _parse_models(payload: object) -> tuple[str, ...]: + if not isinstance(payload, Mapping): + raise ValueError("models response must contain a data list") + data = cast(Mapping[str, object], payload).get("data") + if not isinstance(data, list): + raise ValueError("models response must contain a data list") + models: list[str] = [] + for item in cast(list[object], data): + if isinstance(item, Mapping): + record = cast(Mapping[str, object], item) + if "id" in record: + models.append(str(record["id"])) + return tuple(models) + + +def _resolve_secrets(plan: RunPlan, values: dict[str, str]) -> dict[str, str]: + required = { + variable.source_environment_variable + for variable in plan.command.environment + if isinstance(variable, SecretEnvironmentVariable) + } + missing = sorted(name for name in required if name not in values) + if missing: + raise RuntimeEffectError( + RuntimeDiagnostic( + code="missing-secret", + message=f"secret environment variable {missing[0]!r} is not resolved", + ) + ) + return {name: values[name] for name in required} + + +def _probe_headers(plan: RunPlan, secret_values: Mapping[str, str]) -> dict[str, str]: + source = plan.readiness.bearer_token_environment_variable + if source is None: + return {} + if source not in secret_values: + raise RuntimeEffectError( + RuntimeDiagnostic( + code="missing-secret", + message=f"secret environment variable {source!r} is not resolved", + ) + ) + return {"Authorization": f"Bearer {secret_values[source]}"} + + +def _launch_process( + plan: RunPlan, + argv: tuple[str, ...], + environment: dict[str, str], + log_directory: Path, +) -> LocalProcessHandle: + stdout_path = log_directory / f"{plan.plan_digest}.stdout.log" + stderr_path = log_directory / f"{plan.plan_digest}.stderr.log" + with stdout_path.open("ab") as stdout_file, stderr_path.open("ab") as stderr_file: + process = subprocess.Popen( + argv, + cwd=plan.command.working_directory, + env=environment, + stdout=stdout_file, + stderr=stderr_file, + start_new_session=True, + ) + marker = read_process_start_marker(process.pid) + suffix = marker or "unknown" + return LocalProcessHandle( + external_id=f"{process.pid}:{suffix}", + pid=process.pid, + process_group_id=process.pid, + start_marker=marker, + stdout_path=str(stdout_path), + stderr_path=str(stderr_path), + ) + + +def _launch_docker(argv: tuple[str, ...], environment: dict[str, str]) -> DockerHandle: + completed = subprocess.run(argv, env=environment, capture_output=True, text=True, check=False) + if completed.returncode != 0: + raise RuntimeEffectError( + RuntimeDiagnostic(code="docker-launch-failed", message=completed.stderr.strip() or "docker run failed") + ) + container_id = completed.stdout.strip() + if not container_id: + raise RuntimeEffectError( + RuntimeDiagnostic(code="docker-launch-failed", message="docker run returned no container identity") + ) + return DockerHandle(external_id=container_id, container_id=container_id) + + +def _now() -> str: + return datetime.now(UTC).isoformat() diff --git a/tools/serve_gliner.py b/tools/serve_gliner.py deleted file mode 100644 index 6f8ecaf2..00000000 --- a/tools/serve_gliner.py +++ /dev/null @@ -1,486 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Minimal OpenAI-compatible GLiNER server for Anonymizer detection. - -Fakes /v1/chat/completions so Anonymizer routes its `entity_detector` role -here instead of hitting build.nvidia.com. The response's message.content is -a JSON string of shape ``{"entities": [...]}`` as expected by -``anonymizer.engine.detection.postprocess.parse_raw_entities``. - -Run: - python tools/serve_gliner.py # binds 127.0.0.1:8001 (default) - python tools/serve_gliner.py --port 9000 # override listen port - python tools/serve_gliner.py --host 0.0.0.0 # all interfaces (no auth) - -Chunk batching and entity deduplication are implemented for robust local -inference. This file adds the Anonymizer chat-completion adapter and optional request -coalescing when DataDesigner runs with ``max_parallel_requests`` > 1. - -See ``docs/concepts/self-hosting-gliner.md`` for full usage. -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import logging -import os -import time -import uuid -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass -from typing import Any - -import torch # ty: ignore[unresolved-import] -- optional server dependency -import uvicorn -from fastapi import FastAPI, HTTPException, Request # ty: ignore[unresolved-import] -- optional server dependency -from gliner import GLiNER # ty: ignore[unresolved-import] -- optional server dependency - -MODEL_NAME = "nvidia/gliner-pii" -DEFAULT_HOST = "127.0.0.1" -DEFAULT_PORT = 8001 -DEFAULT_CHUNK_LENGTH = 384 -DEFAULT_OVERLAP = 128 -DEFAULT_FLAT_NER = False -DEFAULT_INFERENCE_BATCH_SIZE = 8 - -BATCH_MODE = os.getenv("GLINER_BATCH_MODE", "true").lower() not in {"0", "false", "no"} -MAX_BATCH_REQUESTS = int(os.getenv("GLINER_MAX_BATCH_REQUESTS", "32")) -BATCH_WAIT_MS = float(os.getenv("GLINER_BATCH_WAIT_MS", "10")) / 1000.0 - -logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") -log = logging.getLogger("gliner-server") - -app = FastAPI() -model: GLiNER | None = None -_device = "cpu" - - -def _resolve_device() -> str: - device_env = os.getenv("DEVICE", "auto") - if device_env != "auto": - return device_env - if torch.backends.mps.is_available(): - return "mps" - if torch.cuda.is_available(): - return "cuda" - return "cpu" - - -def _load_model() -> None: - global model, _device - _device = _resolve_device() - log.info("loading %s on %s", MODEL_NAME, _device) - model = GLiNER.from_pretrained(MODEL_NAME, map_location=_device) - - -@dataclass(frozen=True) -class DetectParams: - labels: tuple[str, ...] - threshold: float - chunk_length: int - overlap: int - flat_ner: bool - inference_batch_size: int - - -@dataclass -class DetectJob: - text: str - params: DetectParams - future: asyncio.Future[list[dict[str, Any]]] - - -def _extract_text(messages: list[dict[str, Any]]) -> str: - """Pull the user message text. Handles both string and multi-part content.""" - if not messages: - return "" - content = messages[-1].get("content", "") - if isinstance(content, str): - return content - if isinstance(content, list): - return "".join(part.get("text", "") for part in content if isinstance(part, dict)) - return str(content) - - -def _validate_chunk_params(chunk_length: int, overlap: int) -> None: - if chunk_length < 1: - raise HTTPException(status_code=422, detail="chunk_length must be >= 1") - if overlap < 0: - raise HTTPException(status_code=422, detail="overlap must be >= 0") - if overlap >= chunk_length: - raise HTTPException(status_code=422, detail="overlap must be less than chunk_length") - - -def _create_text_chunks(text: str, chunk_length: int, overlap: int) -> tuple[list[str], list[int]]: - chunks: list[str] = [] - offsets: list[int] = [] - start = 0 - while start < len(text): - chunks.append(text[start : start + chunk_length]) - offsets.append(start) - if start + chunk_length >= len(text): - break - start += chunk_length - overlap - return chunks, offsets - - -def _shift_offsets(entities: list[dict[str, Any]], offset: int) -> None: - for entity in entities: - entity["start"] = int(entity["start"]) + offset - entity["end"] = int(entity["end"]) + offset - - -def _remove_subset_entities(entities: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Drop spans fully contained in another span (dedup step for nested NER).""" - if not entities: - return [] - - kept = list(entities) - to_delete: list[int] = [] - for idx, ent in enumerate(kept): - has_superset = any( - i != idx - and i not in to_delete - and other["start"] <= ent["start"] - and other["end"] >= ent["end"] - and (other["start"] < ent["start"] or other["end"] > ent["end"]) - for i, other in enumerate(kept) - ) - if has_superset: - to_delete.append(idx) - - for idx in sorted(to_delete, reverse=True): - del kept[idx] - return kept - - -def _dedupe_entities(entities: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Collapse duplicate spans from overlapping chunks, not repeated text elsewhere.""" - best: dict[tuple[str, str, int, int], dict[str, Any]] = {} - for entity in entities: - label = str(entity.get("label", "")) - text = str(entity.get("text", "")) - start = int(entity.get("start", 0)) - end = int(entity.get("end", 0)) - key = (label, text.strip().lower(), start, end) - score = float(entity.get("score", 0.0)) - if key not in best or score > float(best[key].get("score", 0.0)): - best[key] = entity - return list(best.values()) - - -def _format_entities(raw: list[dict[str, Any]]) -> list[dict[str, Any]]: - return [ - { - "text": e["text"], - "label": e["label"], - "start": e["start"], - "end": e["end"], - "score": e["score"], - } - for e in raw - ] - - -def _finalize_entities(entities: list[dict[str, Any]], *, flat_ner: bool) -> list[dict[str, Any]]: - if not entities: - return [] - if flat_ner: - processed = entities - else: - processed = _remove_subset_entities([dict(entity) for entity in entities]) - return _format_entities(_dedupe_entities(processed)) - - -def _run_chunk_inference( - chunks: list[str], - *, - labels: list[str], - threshold: float, - flat_ner: bool, - inference_batch_size: int, -) -> list[list[dict[str, Any]]]: - if model is None: - raise RuntimeError("GLiNER model is not loaded") - if not chunks: - return [] - - batch_entities = model.inference( - texts=chunks, - labels=labels, - threshold=threshold, - flat_ner=flat_ner, - relations=[], - batch_size=inference_batch_size, - ) - if not isinstance(batch_entities, list) or (batch_entities and not isinstance(batch_entities[0], list)): - raise ValueError("unexpected GLiNER inference batch shape") - return batch_entities - - -def _detect_entities_for_text( - text: str, - labels: list[str], - *, - threshold: float, - chunk_length: int, - overlap: int, - flat_ner: bool, - inference_batch_size: int, -) -> list[dict[str, Any]]: - if not labels or not text: - return [] - - chunks, offsets = _create_text_chunks(text, chunk_length, overlap) - batch = _run_chunk_inference( - chunks, - labels=labels, - threshold=threshold, - flat_ner=flat_ner, - inference_batch_size=inference_batch_size, - ) - - merged: list[dict[str, Any]] = [] - for chunk_entities, offset in zip(batch, offsets, strict=True): - adjusted = [dict(entity) for entity in chunk_entities] - _shift_offsets(adjusted, offset) - merged.extend(adjusted) - - return _finalize_entities(merged, flat_ner=flat_ner) - - -def _detect_entities_for_texts( - texts: list[str], - labels: list[str], - *, - threshold: float, - chunk_length: int, - overlap: int, - flat_ner: bool, - inference_batch_size: int, -) -> list[list[dict[str, Any]]]: - if not labels: - return [[] for _ in texts] - - chunk_records: list[tuple[int, int, str]] = [] - for text_idx, text in enumerate(texts): - if not text: - continue - chunks, offsets = _create_text_chunks(text, chunk_length, overlap) - for chunk, offset in zip(chunks, offsets, strict=True): - chunk_records.append((text_idx, offset, chunk)) - - per_text: list[list[dict[str, Any]]] = [[] for _ in texts] - if not chunk_records: - return per_text - - flat_chunks = [chunk for _, _, chunk in chunk_records] - batch = _run_chunk_inference( - flat_chunks, - labels=labels, - threshold=threshold, - flat_ner=flat_ner, - inference_batch_size=inference_batch_size, - ) - - for (text_idx, offset, _), chunk_entities in zip(chunk_records, batch, strict=True): - adjusted = [dict(entity) for entity in chunk_entities] - _shift_offsets(adjusted, offset) - per_text[text_idx].extend(adjusted) - - return [_finalize_entities(entities, flat_ner=flat_ner) for entities in per_text] - - -class BatchDetector: - """Coalesce concurrent detect requests into shared GLiNER inference calls.""" - - def __init__(self) -> None: - self._queue: asyncio.Queue[DetectJob | None] = asyncio.Queue() - self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="gliner-infer") - self._worker_task: asyncio.Task[None] | None = None - - def start(self) -> None: - self._worker_task = asyncio.create_task(self._worker()) - - async def stop(self) -> None: - if self._worker_task is None: - return - await self._queue.put(None) - await self._worker_task - self._executor.shutdown(wait=True) - - async def detect(self, text: str, params: DetectParams) -> list[dict[str, Any]]: - loop = asyncio.get_running_loop() - if not BATCH_MODE: - return await loop.run_in_executor( - self._executor, - lambda: _detect_entities_for_text( - text, - list(params.labels), - threshold=params.threshold, - chunk_length=params.chunk_length, - overlap=params.overlap, - flat_ner=params.flat_ner, - inference_batch_size=params.inference_batch_size, - ), - ) - - future: asyncio.Future[list[dict[str, Any]]] = loop.create_future() - await self._queue.put(DetectJob(text=text, params=params, future=future)) - return await future - - async def _worker(self) -> None: - while True: - first = await self._queue.get() - if first is None: - break - - batch = [first] - deadline = asyncio.get_running_loop().time() + BATCH_WAIT_MS - while len(batch) < MAX_BATCH_REQUESTS: - timeout = deadline - asyncio.get_running_loop().time() - if timeout <= 0: - break - try: - job = await asyncio.wait_for(self._queue.get(), timeout=timeout) - except TimeoutError: - break - if job is None: - await self._queue.put(None) - break - batch.append(job) - - await self._dispatch(batch) - - async def _dispatch(self, jobs: list[DetectJob]) -> None: - grouped: dict[DetectParams, list[DetectJob]] = {} - for job in jobs: - grouped.setdefault(job.params, []).append(job) - - loop = asyncio.get_running_loop() - for params, group in grouped.items(): - texts = [job.text for job in group] - try: - results = await loop.run_in_executor( - self._executor, - lambda p=params, t=texts: _detect_entities_for_texts( - t, - list(p.labels), - threshold=p.threshold, - chunk_length=p.chunk_length, - overlap=p.overlap, - flat_ner=p.flat_ner, - inference_batch_size=p.inference_batch_size, - ), - ) - except Exception as exc: - for job in group: - if not job.future.done(): - job.future.set_exception(exc) - continue - - for job, entities in zip(group, results, strict=True): - if not job.future.done(): - job.future.set_result(entities) - - -detector = BatchDetector() - - -@app.on_event("startup") -async def startup() -> None: - await asyncio.to_thread(_load_model) - log.info( - "device=%s batch_mode=%s max_requests=%d wait_ms=%.0f inference_batch_size=%d", - _device, - BATCH_MODE, - MAX_BATCH_REQUESTS, - BATCH_WAIT_MS * 1000, - DEFAULT_INFERENCE_BATCH_SIZE, - ) - detector.start() - - -@app.on_event("shutdown") -async def shutdown() -> None: - await detector.stop() - - -@app.get("/v1/models") -def list_models() -> dict[str, Any]: - return {"object": "list", "data": [{"id": MODEL_NAME, "object": "model"}]} - - -@app.post("/v1/chat/completions") -async def chat_completions(request: Request) -> dict[str, Any]: - if model is None: - raise HTTPException(status_code=503, detail="GLiNER model is not loaded") - - body = await request.json() - text = _extract_text(body.get("messages", [])) - labels = body.get("labels") or [] - threshold = float(body.get("threshold", 0.3)) - chunk_length = int(body.get("chunk_length", DEFAULT_CHUNK_LENGTH)) - overlap = int(body.get("overlap", DEFAULT_OVERLAP)) - flat_ner = bool(body.get("flat_ner", DEFAULT_FLAT_NER)) - inference_batch_size = int(body.get("batch_size", DEFAULT_INFERENCE_BATCH_SIZE)) - _validate_chunk_params(chunk_length, overlap) - - params = DetectParams( - labels=tuple(labels), - threshold=threshold, - chunk_length=chunk_length, - overlap=overlap, - flat_ner=flat_ner, - inference_batch_size=inference_batch_size, - ) - - log.info( - "detect: labels=%d threshold=%.2f chunk=%d overlap=%d flat_ner=%s batch_size=%d text_len=%d", - len(labels), - threshold, - chunk_length, - overlap, - flat_ner, - inference_batch_size, - len(text), - ) - entities = await detector.detect(text, params) - content = json.dumps({"entities": entities}) - return { - "id": f"chatcmpl-{uuid.uuid4().hex[:12]}", - "object": "chat.completion", - "created": int(time.time()), - "model": body.get("model", MODEL_NAME), - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": content}, - "finish_reason": "stop", - } - ], - "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, - } - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description="OpenAI-compatible GLiNER server for Anonymizer.") - parser.add_argument( - "--host", - default=DEFAULT_HOST, - help=f"Bind address (default: {DEFAULT_HOST})", - ) - parser.add_argument( - "--port", - type=int, - default=DEFAULT_PORT, - help=f"Listen port (default: {DEFAULT_PORT})", - ) - return parser.parse_args() - - -if __name__ == "__main__": - cli = _parse_args() - uvicorn.run(app, host=cli.host, port=cli.port) diff --git a/tools/vllm_debug.py b/tools/vllm_debug.py deleted file mode 100644 index 593b5040..00000000 --- a/tools/vllm_debug.py +++ /dev/null @@ -1,290 +0,0 @@ -#!/usr/bin/env python -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Launch and probe local vLLM servers for Anonymizer development. - -Usage: - uv run python tools/vllm_debug.py models --cached - uv run --with vllm python tools/vllm_debug.py serve /path/to/model --dry-run - uv run python tools/vllm_debug.py serve /path/to/model --vllm-python /opt/vllm/bin/python - uv run python tools/vllm_debug.py models --endpoint http://127.0.0.1:8000/v1 - uv run python tools/vllm_debug.py call --model local-model --prompt "Hello" - -The helper does not prefetch models. ``serve`` requires vLLM to be installed in -the Python environment that launches this script. Use a cached snapshot path -from ``models --cached`` to avoid a Hugging Face download; a model ID may cause -vLLM to download it. -""" - -from __future__ import annotations - -import json -import os -import shlex -import subprocess -import sys -from pathlib import Path -from typing import Annotated, Any - -import cyclopts -import httpx -from pydantic import BaseModel - -app = cyclopts.App(help=__doc__) - -DEFAULT_ENDPOINT = "http://127.0.0.1:8000/v1" - - -class ServeRequest(BaseModel): - """Arguments for an OpenAI-compatible vLLM server.""" - - model: str - host: str = "127.0.0.1" - port: int = 8000 - adapter: Path | None = None - adapter_name: str | None = None - tensor_parallel_size: int | None = None - gpu_memory_utilization: float | None = None - max_model_len: int | None = None - eager: bool = False - python_executable: Path | None = None - served_model_name: str | None = None - api_key: str | None = None - - -class CachedModel(BaseModel): - """A Hugging Face cache snapshot usable as a local vLLM model path.""" - - repository: str - snapshot_path: Path - - -class CallResult(BaseModel): - """Normalized OpenAI-compatible chat response.""" - - content: str - usage: dict[str, Any] - - -def build_serve_command(request: ServeRequest) -> list[str]: - """Build the vLLM server command without starting a process.""" - command = [ - str(request.python_executable or sys.executable), - "-m", - "vllm.entrypoints.openai.api_server", - "--model", - request.model, - "--host", - request.host, - "--port", - str(request.port), - ] - if request.served_model_name is not None: - command.extend(["--served-model-name", request.served_model_name]) - if request.api_key is not None: - command.extend(["--api-key", request.api_key]) - _append_adapter(command, request) - _append_gpu_options(command, request) - return command - - -def render_serve_command(command: list[str]) -> str: - """Render a server command without exposing its API key.""" - rendered = command.copy() - if "--api-key" in rendered: - key_index = rendered.index("--api-key") + 1 - if key_index < len(rendered): - rendered[key_index] = "" - return shlex.join(rendered) - - -def _append_adapter(command: list[str], request: ServeRequest) -> None: - if request.adapter is None: - return - adapter_name = request.adapter_name or request.adapter.name - command.extend(["--enable-lora", "--lora-modules", f"{adapter_name}={request.adapter}"]) - - -def _append_gpu_options(command: list[str], request: ServeRequest) -> None: - options = [ - ("--tensor-parallel-size", request.tensor_parallel_size), - ("--gpu-memory-utilization", request.gpu_memory_utilization), - ("--max-model-len", request.max_model_len), - ] - for flag, value in options: - if value is not None: - command.extend([flag, str(value)]) - if request.eager: - command.append("--enforce-eager") - - -def discover_cached_models(cache_root: Path) -> list[CachedModel]: - """Return all snapshot directories in a Hugging Face hub cache.""" - if not cache_root.exists(): - return [] - models: list[CachedModel] = [] - for model_dir in sorted(cache_root.glob("models--*")): - repository = model_dir.name.removeprefix("models--").replace("--", "/") - for snapshot in sorted((model_dir / "snapshots").glob("*")): - if snapshot.is_dir(): - models.append(CachedModel(repository=repository, snapshot_path=snapshot)) - return models - - -def default_cache_root() -> Path: - """Resolve the Hugging Face hub cache without creating it.""" - if hub_cache := os.getenv("HF_HUB_CACHE"): - return Path(hub_cache) - if hf_home := os.getenv("HF_HOME"): - return Path(hf_home) / "hub" - return Path.home() / ".cache" / "huggingface" / "hub" - - -def fetch_models(endpoint: str, *, timeout_seconds: float) -> list[str]: - """Fetch model IDs from an OpenAI-compatible ``/v1/models`` endpoint.""" - response = httpx.get(f"{normalize_endpoint(endpoint)}/models", timeout=timeout_seconds) - response.raise_for_status() - payload = response.json() - return [str(item["id"]) for item in payload.get("data", []) if isinstance(item, dict) and "id" in item] - - -def call_chat( - *, - endpoint: str, - model: str, - prompt: str, - timeout_seconds: float, - api_key: str | None, -) -> CallResult: - """Send a single chat completion request and normalize the response.""" - headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} - payload = {"model": model, "messages": [{"role": "user", "content": prompt}]} - response = httpx.post( - f"{normalize_endpoint(endpoint)}/chat/completions", - json=payload, - timeout=timeout_seconds, - headers=headers, - ) - response.raise_for_status() - body = response.json() - content = str(body["choices"][0]["message"].get("content", "")) - usage = body.get("usage", {}) - return CallResult(content=content, usage=usage if isinstance(usage, dict) else {}) - - -def normalize_endpoint(endpoint: str) -> str: - """Ensure an endpoint has one ``/v1`` suffix and no trailing slash.""" - stripped = endpoint.rstrip("/") - return stripped if stripped.endswith("/v1") else f"{stripped}/v1" - - -def resolve_api_key(api_key: str | None, api_key_env: str | None) -> str | None: - """Return an explicit API key or one read from a named environment variable.""" - if api_key is not None and api_key_env is not None: - raise ValueError("Use either api_key or api_key_env, not both.") - if api_key_env is None: - return api_key - value = os.getenv(api_key_env) - if value is None: - raise ValueError(f"Environment variable {api_key_env!r} is not set.") - return value - - -def render(value: BaseModel | list[str] | list[CachedModel], *, json_output: bool) -> str: - """Render command results in JSON or compact human-readable text.""" - if json_output: - return json.dumps(_json_value(value), indent=2) - if isinstance(value, list): - return "\n".join(str(item) for item in value) or "No models found." - return value.model_dump_json(indent=2) - - -def _json_value(value: BaseModel | list[str] | list[CachedModel]) -> Any: - if isinstance(value, BaseModel): - return value.model_dump(mode="json") - return [item.model_dump(mode="json") if isinstance(item, BaseModel) else item for item in value] - - -@app.command -def serve( - model: str, - *, - host: str = "127.0.0.1", - port: int = 8000, - adapter: Path | None = None, - adapter_name: str | None = None, - tensor_parallel_size: int | None = None, - gpu_memory_utilization: float | None = None, - max_model_len: int | None = None, - eager: bool = False, - vllm_python: Annotated[Path | None, cyclopts.Parameter("--vllm-python")] = None, - served_model_name: str | None = None, - api_key: str | None = None, - api_key_env: str | None = None, - dry_run: Annotated[bool, cyclopts.Parameter("--dry-run")] = False, -) -> None: - """Launch an OpenAI-compatible vLLM server from the current Python environment.""" - request = ServeRequest( - model=model, - host=host, - port=port, - adapter=adapter, - adapter_name=adapter_name, - tensor_parallel_size=tensor_parallel_size, - gpu_memory_utilization=gpu_memory_utilization, - max_model_len=max_model_len, - eager=eager, - python_executable=vllm_python, - served_model_name=served_model_name, - api_key=resolve_api_key(api_key, api_key_env), - ) - command = build_serve_command(request) - if dry_run: - print(render_serve_command(command)) - return - try: - subprocess.run(command, check=True) - except FileNotFoundError as exc: - raise SystemExit(f"vLLM Python executable not found: {command[0]}") from exc - - -@app.command -def models( - *, - endpoint: str = DEFAULT_ENDPOINT, - cached: Annotated[bool, cyclopts.Parameter("--cached")] = False, - cache_root: Path | None = None, - timeout_seconds: float = 10.0, - json_output: Annotated[bool, cyclopts.Parameter("--json")] = False, -) -> None: - """List served models or cached Hugging Face snapshots.""" - if cached: - print(render(discover_cached_models(cache_root or default_cache_root()), json_output=json_output)) - return - print(render(fetch_models(endpoint, timeout_seconds=timeout_seconds), json_output=json_output)) - - -@app.command -def call( - model: str, - prompt: str, - *, - endpoint: str = DEFAULT_ENDPOINT, - api_key_env: str | None = None, - timeout_seconds: float = 60.0, - json_output: Annotated[bool, cyclopts.Parameter("--json")] = False, -) -> None: - """Send one OpenAI-compatible chat completion request.""" - api_key = resolve_api_key(None, api_key_env) - result = call_chat( - endpoint=endpoint, - model=model, - prompt=prompt, - timeout_seconds=timeout_seconds, - api_key=api_key, - ) - print(render(result, json_output=json_output)) - - -if __name__ == "__main__": - app() From 18edc2cdd05efd937c412ea475f24d95709c9b7e Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Mon, 10 Aug 2026 22:31:39 +0000 Subject: [PATCH 03/28] feat(dev): add TOML inference profiles and vLLM factory Signed-off-by: Aaron Gonzales --- README.md | 4 +- docs/concepts/inference-services.md | 145 +++---- docs/concepts/self-hosting-gliner.md | 4 +- .../posts/self-hosted-anonymizer-b300.md | 45 +- pyproject.toml | 2 +- tests/tools/test_inference_service.py | 236 ++++++++++- tests/tools/test_vllm_factory.py | 153 +++++++ tools/inference_service_compiler/cli.py | 11 +- tools/inference_service_compiler/compiler.py | 15 +- tools/inference_service_compiler/models.py | 2 +- .../native_gliner.py | 74 ++-- tools/inference_service_compiler/profiles.py | 16 + tools/inference_service_compiler/runtime.py | 11 +- .../vllm_factory.py | 133 ++++++ .../inference_service_compiler/vllm_server.py | 16 + tools/inference_service_profiles/gliner2.toml | 30 ++ .../nvidia-gliner.toml | 30 ++ .../vllm-local.toml | 30 ++ uv.lock | 384 +++++++++++++----- 19 files changed, 1055 insertions(+), 286 deletions(-) create mode 100644 tests/tools/test_vllm_factory.py create mode 100644 tools/inference_service_compiler/profiles.py create mode 100644 tools/inference_service_compiler/vllm_factory.py create mode 100644 tools/inference_service_compiler/vllm_server.py create mode 100644 tools/inference_service_profiles/gliner2.toml create mode 100644 tools/inference_service_profiles/nvidia-gliner.toml create mode 100644 tools/inference_service_profiles/vllm-local.toml diff --git a/README.md b/README.md index 38350939..6088d38a 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ Use the source-tree inference service compiler to create immutable plans and managed launch receipts for native GLiNER or vLLM processes and containers: ```bash -uv run tools/inference_service.py compile --intent intent.json --source-revision 3f68c145 --output plan.json +uv run tools/inference_service.py compile --profile tools/inference_service_profiles/nvidia-gliner.toml --source-revision 3f68c145 --output plan.json uv run tools/inference_service.py launch --plan plan.json --output launch.json uv run tools/inference_service.py inspect --receipt launch.json uv run tools/inference_service.py cancel --receipt launch.json @@ -168,7 +168,7 @@ uv run tools/inference_service.py cancel --receipt launch.json The tool is not part of the wheel and does not attach to externally owned endpoints. The [local inference service guide](docs/concepts/inference-services.md) -covers typed intents, GPU-host setup, Docker, model discovery, capability +covers typed TOML profiles, GPU-host setup, Docker, model discovery, capability probes, and Anonymizer provider configuration. --- diff --git a/docs/concepts/inference-services.md b/docs/concepts/inference-services.md index 62022b46..4805f043 100644 --- a/docs/concepts/inference-services.md +++ b/docs/concepts/inference-services.md @@ -4,7 +4,7 @@ # Run Local Inference Services Anonymizer's source tree includes an inference service compiler for development -and controlled internal deployments. It compiles typed JSON intent into an +and controlled internal deployments. It compiles a typed TOML profile into an immutable plan before it starts a process or container. Launch, probe, inspect, and cancel operations return versioned JSON receipts with the external identity and known effects of the operation. @@ -13,7 +13,7 @@ The tool is source-owned under `tools/` and is not included in the `nemo-anonymizer` wheel. It currently supports: - entity detection with the native NVIDIA GLiNER or GLiNER2 runtime; -- entity detection or generation through a vLLM-compatible model; +- generation through a vLLM-compatible model; - managed local processes and managed local Docker containers; and - direct HTTP access to the resulting OpenAI-compatible endpoint. @@ -26,9 +26,9 @@ service. The CLI keeps compilation separate from runtime effects: ```text -intent.json -> compile -> plan.json -> launch -> launch.json - inspect <-+ - cancel <-+ +profile.toml -> compile -> plan.json -> launch -> launch.json + inspect <-+ + cancel <-+ ``` Plans use the `inference-service.run-plan/v1` schema. Launch, capability-probe, @@ -39,46 +39,13 @@ performing effects. ## Native GLiNER -Create `gliner-intent.json`: - -```json -{ - "schema_version": "inference-service.intent/v1", - "task": { - "kind": "entity-detection", - "dynamic_labels": true, - "offsets": true, - "scores": true - }, - "model": { - "kind": "hugging-face", - "model_id": "nvidia/gliner-pii" - }, - "engine": { - "kind": "native-gliner", - "family": "nvidia-gliner", - "device": "auto" - }, - "placement": { - "kind": "local-process", - "host": "127.0.0.1", - "port": 8001 - }, - "access": {"kind": "direct"}, - "lifecycle": { - "kind": "managed", - "startup_timeout_seconds": 120, - "shutdown_timeout_seconds": 30 - } -} -``` - -Compile and launch it from the repository root. Replace the example source -revision with the revision of your checkout: +The source tree includes pinned profiles for NVIDIA GLiNER and GLiNER2 under +`tools/inference_service_profiles/`. Compile and launch one from the repository +root. Replace the example source revision with the revision of your checkout: ```bash uv run tools/inference_service.py compile \ - --intent gliner-intent.json \ + --profile tools/inference_service_profiles/nvidia-gliner.toml \ --source-revision 3f68c145 \ --output gliner-plan.json @@ -91,9 +58,10 @@ The launch returns only after `/v1/models` and an entity-detection contract probe succeed. The receipt records the process ID plus its Linux start marker when available, which lets a later invocation guard against PID reuse. -To use GLiNER2, change the engine family to `gliner2` and select a compatible -checkpoint such as `fastino/gliner2-privacy-filter-PII-multi` in the model -field. `nvidia-gliner` and `nvidia/gliner-pii` remain the defaults. +Use `tools/inference_service_profiles/gliner2.toml` for the pinned GLiNER2 +checkpoint. Entity detection is intentionally native. The compiler rejects a +GLiNER profile paired with vLLM because vLLM 0.26 does not expose GLiNER's +dynamic-label, offset, and score contract. ## Local vLLM Process @@ -104,42 +72,27 @@ uv sync --group dev --group local-models nvidia-smi ``` -Create an intent with a generation task, vLLM engine, and local-process -placement: - -```json -{ - "schema_version": "inference-service.intent/v1", - "task": {"kind": "generation", "chat": true}, - "model": { - "kind": "hugging-face", - "model_id": "openai/gpt-oss-20b", - "revision": "PIN_A_MODEL_REVISION_HERE" - }, - "engine": { - "kind": "vllm", - "executable": ".venv/bin/vllm", - "served_model_name": "anonymizer-local", - "gpu_memory_utilization": 0.85, - "max_model_len": 8192 - }, - "placement": { - "kind": "local-process", - "host": "127.0.0.1", - "port": 8000 - }, - "access": {"kind": "direct"}, - "lifecycle": { - "kind": "managed", - "startup_timeout_seconds": 600, - "shutdown_timeout_seconds": 30 - } -} +The `local-models` group pins vLLM 0.26.0. The local plan starts +`tools/inference_service_compiler/vllm_server.py`, which constructs vLLM's +frontend and async engine through its Python API. It does not invoke `vllm +serve` or inherit vLLM's full CLI surface. + +Compile `tools/inference_service_profiles/vllm-local.toml`, or copy it and pin +the model revision and sizing fields for your workload: + +```bash +uv run tools/inference_service.py compile \ + --profile tools/inference_service_profiles/vllm-local.toml \ + --source-revision 3f68c145 \ + --output vllm-plan.json + +uv run tools/inference_service.py launch \ + --plan vllm-plan.json \ + --output vllm-launch.json ``` -Use the same `compile` and `launch` commands shown for GLiNER. The model ID may -cause vLLM to download weights. List existing Hugging Face cache snapshots -without downloading anything: +The model ID may cause vLLM to download weights. List existing Hugging Face +cache snapshots without downloading anything: ```bash uv run tools/inference_service.py models --output cached-models.json @@ -147,11 +100,10 @@ uv run tools/inference_service.py models --output cached-models.json Add a LoRA artifact to the model when needed: -```json -"adapter": { - "path": "/models/privacy-adapter", - "name": "privacy" -} +```toml +[model.adapter] +path = "/models/privacy-adapter" +name = "privacy" ``` The compiler renders the corresponding vLLM `--lora-modules` arguments. @@ -160,16 +112,15 @@ The compiler renders the corresponding vLLM `--lora-modules` arguments. Change the placement to Docker to use vLLM's official OpenAI-compatible image: -```json -"placement": { - "kind": "docker", - "host": "127.0.0.1", - "port": 8000, - "image": "vllm/vllm-openai:v0.20.0", - "runtime": "docker", - "gpus": "all", - "hugging_face_cache": "/home/user/.cache/huggingface" -} +```toml +[placement] +kind = "docker" +host = "127.0.0.1" +port = 8000 +image = "vllm/vllm-openai:v0.26.0" +runtime = "docker" +gpus = "all" +hugging_face_cache = "/home/user/.cache/huggingface" ``` The plan records the exact image and complete `docker run` argv. Launch receipts @@ -180,8 +131,10 @@ Pin an image version appropriate for the host's driver and CUDA compatibility. Set `api_key_env` on the vLLM engine to reference a named environment variable: -```json -"api_key_env": "LOCAL_VLLM_API_KEY" +```toml +[engine] +kind = "vllm" +api_key_env = "LOCAL_VLLM_API_KEY" ``` Plans serialize only that source name and render the service environment value diff --git a/docs/concepts/self-hosting-gliner.md b/docs/concepts/self-hosting-gliner.md index 56dbf26f..9aaef969 100644 --- a/docs/concepts/self-hosting-gliner.md +++ b/docs/concepts/self-hosting-gliner.md @@ -94,9 +94,9 @@ On first launch, the selected public checkpoint is downloaded from Hugging Face ### Start the server -Create a native GLiNER intent, compile it, and launch the resulting plan as +Compile the pinned native GLiNER TOML profile and launch the resulting plan as shown in [Run local inference services](inference-services.md#native-gliner). -The intent keeps the model checkpoint, engine family, device, placement, +The profile keeps the model checkpoint, engine family, device, placement, access, and managed lifecycle separate. `nvidia-gliner` is the default engine family and `nvidia/gliner-pii` is the default model; GLiNER2 uses the `fastino/gliner2-privacy-filter-PII-multi` checkpoint. diff --git a/docs/devnotes/posts/self-hosted-anonymizer-b300.md b/docs/devnotes/posts/self-hosted-anonymizer-b300.md index 92bb9c39..ac7ee1f8 100644 --- a/docs/devnotes/posts/self-hosted-anonymizer-b300.md +++ b/docs/devnotes/posts/self-hosted-anonymizer-b300.md @@ -138,21 +138,44 @@ service compiler described in [Self-hosting GLiNER](../../concepts/self-hosting- which records the exact model, engine, placement, batch environment, endpoint, and process identity in versioned plans and receipts. -```json title="gliner-b300-intent.json" -{ - "schema_version": "inference-service.intent/v1", - "task": {"kind": "entity-detection", "dynamic_labels": true, "offsets": true, "scores": true}, - "model": {"kind": "hugging-face", "model_id": "nvidia/gliner-pii", "revision": "bd23e8ef4425fd04e34c5204ab49ffaa706eae79"}, - "engine": {"kind": "native-gliner", "family": "nvidia-gliner", "device": "cuda", "max_batch_requests": 64, "batch_wait_ms": 10}, - "placement": {"kind": "local-process", "host": "127.0.0.1", "port": 9000}, - "access": {"kind": "direct"}, - "lifecycle": {"kind": "managed", "startup_timeout_seconds": 300, "shutdown_timeout_seconds": 30} -} +```toml title="gliner-b300.toml" +schema_version = "inference-service.intent/v1" + +[task] +kind = "entity-detection" +dynamic_labels = true +offsets = true +scores = true + +[model] +kind = "hugging-face" +model_id = "nvidia/gliner-pii" +revision = "bd23e8ef4425fd04e34c5204ab49ffaa706eae79" + +[engine] +kind = "native-gliner" +family = "nvidia-gliner" +device = "cuda" +max_batch_requests = 64 +batch_wait_ms = 10 + +[placement] +kind = "local-process" +host = "127.0.0.1" +port = 9000 + +[access] +kind = "direct" + +[lifecycle] +kind = "managed" +startup_timeout_seconds = 300 +shutdown_timeout_seconds = 30 ``` ```bash uv run tools/inference_service.py compile \ - --intent gliner-b300-intent.json \ + --profile gliner-b300.toml \ --source-revision 3f68c145 \ --output gliner-b300-plan.json uv run tools/inference_service.py launch \ diff --git a/pyproject.toml b/pyproject.toml index 9c51f894..32a66c85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ notebooks = [ "pillow>=12.0.0,<13", ] local-models = [ - "vllm==0.20.0; sys_platform == 'linux'", + "vllm==0.26.0; sys_platform == 'linux'", ] [build-system] diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index 2ae834f1..fc171ce9 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -15,11 +15,14 @@ import httpx import pytest +from pydantic import ValidationError REPO_ROOT = Path(__file__).resolve().parents[2] CLI_PATH = REPO_ROOT / "tools" / "inference_service.py" TOOLS_ROOT = REPO_ROOT / "tools" NATIVE_GLINER_PATH = TOOLS_ROOT / "inference_service_compiler" / "native_gliner.py" +VLLM_SERVER_PATH = TOOLS_ROOT / "inference_service_compiler" / "vllm_server.py" +PROFILE_ROOT = TOOLS_ROOT / "inference_service_profiles" REMOVED_GLINER_PATH = TOOLS_ROOT / "serve_gliner.py" @@ -68,7 +71,7 @@ def build_generation_plan( models.DockerPlacement( host="127.0.0.1", port=8000, - image="vllm/vllm-openai:v0.20.0", + image="vllm/vllm-openai:v0.26.0", gpus="all", ) if docker @@ -151,7 +154,7 @@ def test_compile_vllm_docker_plan_keeps_secrets_symbolic() -> None: placement=models.DockerPlacement( host="127.0.0.1", port=8000, - image="vllm/vllm-openai:v0.20.0", + image="vllm/vllm-openai:v0.26.0", gpus="all", ), access=models.DirectAccess(), @@ -162,7 +165,7 @@ def test_compile_vllm_docker_plan_keeps_secrets_symbolic() -> None: rendered = plan.model_dump_json() assert plan.runtime.kind == "docker" - assert plan.runtime.image == "vllm/vllm-openai:v0.20.0" + assert plan.runtime.image == "vllm/vllm-openai:v0.26.0" assert plan.endpoint.url == "http://127.0.0.1:8000/v1" assert plan.expected_model == "anonymizer-local" assert plan.required_capabilities == ("chat-completions",) @@ -193,6 +196,51 @@ def test_compile_vllm_docker_plan_keeps_secrets_symbolic() -> None: ) +def test_compile_local_vllm_plan_uses_the_python_server_factory() -> None: + """Local vLLM runs through the source-owned Python factory, not its CLI binary.""" + models, compiler = load_compiler_modules() + intent = models.InferenceIntent( + task=models.Generation(chat=True), + model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b", revision="abcdef0123456789"), + engine=models.VllmEngine(python_executable=".venv/bin/python"), + placement=models.LocalProcessPlacement(host="127.0.0.1", port=8000), + access=models.DirectAccess(), + lifecycle=models.ManagedLifecycle(), + ) + + plan = compiler.compile_intent(intent, source_revision="3f68c145") + + assert plan.command.render_argv()[:3] == ( + ".venv/bin/python", + "tools/inference_service_compiler/vllm_server.py", + "openai/gpt-oss-20b", + ) + assert "serve" not in plan.command.render_argv() + assert VLLM_SERVER_PATH.is_file() + + +def test_compiler_rejects_gliner_through_vllm() -> None: + """GLiNER models stay on their characterized native runtime.""" + models, compiler = load_compiler_modules() + intent = models.InferenceIntent( + task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), + model=models.HuggingFaceModel(model_id="nvidia/gliner-pii"), + engine=models.VllmEngine(), + placement=models.LocalProcessPlacement(), + access=models.DirectAccess(), + lifecycle=models.ManagedLifecycle(), + ) + + with pytest.raises(compiler.CompilationError) as exc_info: + compiler.compile_intent(intent, source_revision="3f68c145") + + assert exc_info.value.diagnostic.code == "unsupported-task-engine" + assert exc_info.value.diagnostic.details == { + "engine": "vllm", + "task": "entity-detection", + } + + def test_compiler_rejects_unsupported_native_generation() -> None: """Compatibility failures are typed compiler diagnostics, not runtime surprises.""" models, compiler = load_compiler_modules() @@ -255,23 +303,37 @@ def test_plan_digest_detects_transport_mutation() -> None: compiler.load_plan(json.dumps(payload)) -def test_compile_command_writes_the_versioned_plan(tmp_path: Path) -> None: - """The CLI is a thin JSON transport over the same pure compiler.""" +def test_compile_command_accepts_toml_profile_and_writes_json_plan(tmp_path: Path) -> None: + """Operators author TOML while generated plans retain the JSON transport.""" cli = load_cli_module() - intent_path = tmp_path / "intent.json" + profile_path = tmp_path / "generation.toml" plan_path = tmp_path / "plan.json" - intent_path.write_text( - json.dumps( - { - "schema_version": "inference-service.intent/v1", - "task": {"kind": "generation", "chat": True}, - "model": {"kind": "hugging-face", "model_id": "openai/gpt-oss-20b"}, - "engine": {"kind": "vllm"}, - "placement": {"kind": "local-process", "host": "127.0.0.1", "port": 8000}, - "access": {"kind": "direct"}, - "lifecycle": {"kind": "managed"}, - } - ), + profile_path.write_text( + """\ +schema_version = "inference-service.intent/v1" + +[task] +kind = "generation" +chat = true + +[model] +kind = "hugging-face" +model_id = "openai/gpt-oss-20b" + +[engine] +kind = "vllm" + +[placement] +kind = "local-process" +host = "127.0.0.1" +port = 8000 + +[access] +kind = "direct" + +[lifecycle] +kind = "managed" +""", encoding="utf-8", ) @@ -280,8 +342,8 @@ def test_compile_command_writes_the_versioned_plan(tmp_path: Path) -> None: cli.app( [ "compile", - "--intent", - str(intent_path), + "--profile", + str(profile_path), "--source-revision", "3f68c145", "--output", @@ -296,6 +358,112 @@ def test_compile_command_writes_the_versioned_plan(tmp_path: Path) -> None: assert payload["plan_digest"] +def test_equivalent_toml_profiles_compile_to_the_same_plan(tmp_path: Path) -> None: + """TOML comments and table order do not change semantic plan identity.""" + cli = load_cli_module() + _models, compiler = load_compiler_modules() + first_path = tmp_path / "first.toml" + second_path = tmp_path / "second.toml" + first_path.write_text( + """\ +schema_version = "inference-service.intent/v1" +[task] +kind = "generation" +chat = true +[model] +kind = "hugging-face" +model_id = "openai/gpt-oss-20b" +[engine] +kind = "vllm" +[placement] +kind = "local-process" +host = "127.0.0.1" +port = 8000 +[access] +kind = "direct" +[lifecycle] +kind = "managed" +""", + encoding="utf-8", + ) + second_path.write_text( + """\ +# The order and formatting are for humans; the compiler sees one typed value. +schema_version = "inference-service.intent/v1" +[lifecycle] +kind = "managed" +[access] +kind = "direct" +[placement] +port = 8000 +host = "127.0.0.1" +kind = "local-process" +[engine] +kind = "vllm" +[model] +model_id = "openai/gpt-oss-20b" +kind = "hugging-face" +[task] +chat = true +kind = "generation" +""", + encoding="utf-8", + ) + + first = compiler.compile_intent(cli.load_profile(first_path), source_revision="3f68c145") + second = compiler.compile_intent(cli.load_profile(second_path), source_revision="3f68c145") + + assert first == second + + +def test_toml_profile_rejects_unknown_engine_fields(tmp_path: Path) -> None: + """Closed profile tables reject unknown settings before compilation.""" + cli = load_cli_module() + profile_path = tmp_path / "invalid.toml" + profile_path.write_text( + """\ +schema_version = "inference-service.intent/v1" +[task] +kind = "generation" +chat = true +[model] +kind = "hugging-face" +model_id = "openai/gpt-oss-20b" +[engine] +kind = "vllm" +unknown = true +[placement] +kind = "local-process" +[access] +kind = "direct" +[lifecycle] +kind = "managed" +""", + encoding="utf-8", + ) + + with pytest.raises(ValidationError): + cli.load_profile(profile_path) + + +def test_reference_toml_profiles_are_pinned_and_compile() -> None: + """Bundled operator profiles stay parseable, pinned, and compatible.""" + cli = load_cli_module() + _models, compiler = load_compiler_modules() + + profile_paths = tuple(sorted(PROFILE_ROOT.glob("*.toml"))) + + assert {path.name for path in profile_paths} == { + "gliner2.toml", + "nvidia-gliner.toml", + "vllm-local.toml", + } + for path in profile_paths: + intent = cli.load_profile(path) + assert intent.model.revision is not None + assert compiler.compile_intent(intent, source_revision="3f68c145") + + def test_probe_records_generation_capabilities() -> None: """A live probe records models and observed task capabilities in a v1 receipt.""" models, compiler = load_compiler_modules() @@ -400,6 +568,7 @@ def test_launch_local_process_returns_reconnectable_handle(tmp_path: Path) -> No mock.patch.object(runtime.subprocess, "Popen", return_value=process) as popen, mock.patch.object(runtime, "probe_endpoint", return_value=probe), mock.patch.object(runtime, "read_process_start_marker", return_value="100"), + mock.patch.object(runtime, "is_handle_running", return_value=True), ): receipt = runtime.launch_plan( plan, @@ -467,6 +636,7 @@ def test_launch_docker_returns_container_identity(tmp_path: Path) -> None: with ( mock.patch.object(runtime.subprocess, "run", return_value=completed) as run, mock.patch.object(runtime, "probe_endpoint", return_value=probe), + mock.patch.object(runtime, "is_handle_running", return_value=True), ): receipt = runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) @@ -658,3 +828,29 @@ def test_failed_readiness_cleans_up_the_launched_process(tmp_path: Path) -> None assert exc_info.value.diagnostic.known_effects == ("4242:100",) assert exc_info.value.diagnostic.cleanup_complete is True killpg.assert_called_once_with(4242, runtime.signal.SIGTERM) + + +def test_readiness_stops_polling_when_the_managed_process_exits(tmp_path: Path) -> None: + """A crashed server fails immediately and points the operator to its stderr log.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + plan = build_generation_plan(models, compiler) + handle = models.LocalProcessHandle( + external_id="4242:100", + pid=4242, + process_group_id=4242, + start_marker="100", + stdout_path=str(tmp_path / "stdout.log"), + stderr_path=str(tmp_path / "stderr.log"), + ) + + with ( + mock.patch.object(runtime, "is_handle_running", return_value=False), + mock.patch.object(runtime, "probe_endpoint") as probe, + pytest.raises(runtime.RuntimeEffectError) as exc_info, + ): + runtime.wait_for_readiness(plan, handle=handle) + + assert exc_info.value.diagnostic.code == "launch-exited" + assert str(tmp_path / "stderr.log") in exc_info.value.diagnostic.message + probe.assert_not_called() diff --git a/tests/tools/test_vllm_factory.py b/tests/tools/test_vllm_factory.py new file mode 100644 index 00000000..31347a52 --- /dev/null +++ b/tests/tools/test_vllm_factory.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the local vLLM Python construction boundary.""" + +from __future__ import annotations + +import importlib +import os +import sys +import tomllib +from pathlib import Path +from unittest import mock + +import pytest + +TOOLS_ROOT = Path(__file__).resolve().parents[2] / "tools" +REPO_ROOT = TOOLS_ROOT.parent + + +def test_local_models_group_pins_vllm_0_26() -> None: + """The characterized Python factory stays bound to the reviewed vLLM release.""" + project = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) + + assert project["dependency-groups"]["local-models"] == ["vllm==0.26.0; sys_platform == 'linux'"] + + +def load_factory_module(): + """Load the source-tree factory without packaging it.""" + sys.path.insert(0, str(TOOLS_ROOT)) + try: + return importlib.import_module("inference_service_compiler.vllm_factory") + finally: + sys.path.pop(0) + + +def test_parse_server_parameters_accepts_only_the_compiler_contract() -> None: + """The process entry point accepts the bounded arguments emitted by the compiler.""" + factory = load_factory_module() + + parameters = factory.parse_server_parameters( + [ + "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + "--host", + "127.0.0.1", + "--port", + "8123", + "--revision", + "fe8a4ea1", + "--tokenizer-revision", + "fe8a4ea1", + "--served-model-name", + "tiny", + "--tensor-parallel-size", + "2", + "--gpu-memory-utilization", + "0.8", + "--max-model-len", + "2048", + "--enforce-eager", + ] + ) + + assert parameters.model == "TinyLlama/TinyLlama-1.1B-Chat-v1.0" + assert parameters.port == 8123 + assert parameters.tensor_parallel_size == 2 + assert parameters.gpu_memory_utilization == 0.8 + assert parameters.enforce_eager is True + + +def test_factory_constructs_vllm_frontend_and_async_engine_arguments() -> None: + """The local service is built from vLLM Python config objects.""" + pytest.importorskip("vllm") + factory = load_factory_module() + parameters = factory.VllmServerParameters( + model="TinyLlama/TinyLlama-1.1B-Chat-v1.0", + host="127.0.0.1", + port=8123, + revision="fe8a4ea1", + tokenizer_revision="fe8a4ea1", + served_model_name="tiny", + tensor_parallel_size=2, + gpu_memory_utilization=0.8, + max_model_len=2048, + enforce_eager=True, + lora_module="privacy=/models/privacy-adapter", + ) + + arguments = factory.build_server_arguments(parameters) + + assert arguments.model == parameters.model + assert arguments.host == "127.0.0.1" + assert arguments.port == 8123 + assert arguments.revision == "fe8a4ea1" + assert arguments.tokenizer_revision == "fe8a4ea1" + assert arguments.served_model_name == ["tiny"] + assert arguments.tensor_parallel_size == 2 + assert arguments.gpu_memory_utilization == 0.8 + assert arguments.max_model_len == 2048 + assert arguments.enforce_eager is True + assert arguments.enable_lora is True + assert [(module.name, module.path) for module in arguments.lora_modules] == [("privacy", "/models/privacy-adapter")] + + +def test_run_server_uses_the_vllm_0_26_lifecycle_boundary() -> None: + """The process runner imports and invokes vLLM 0.26's relocated setup API.""" + pytest.importorskip("vllm") + uvloop = importlib.import_module("uvloop") + api_server = importlib.import_module("vllm.entrypoints.openai.api_server") + api_utils = importlib.import_module("vllm.entrypoints.serve.utils.api_utils") + + factory = load_factory_module() + arguments = mock.sentinel.arguments + coroutine = mock.sentinel.coroutine + run_vllm_server = mock.Mock(return_value=coroutine) + + with ( + mock.patch.object(factory, "parse_server_parameters", return_value=mock.sentinel.parameters), + mock.patch.object(factory, "build_server_arguments", return_value=arguments), + mock.patch.object(api_utils, "cli_env_setup") as cli_env_setup, + mock.patch.object(api_server, "run_server", new=run_vllm_server), + mock.patch.object(uvloop, "run") as uvloop_run, + ): + factory.run_server(["model", "--host", "127.0.0.1", "--port", "8000"]) + + cli_env_setup.assert_called_once_with() + run_vllm_server.assert_called_once_with(arguments) + uvloop_run.assert_called_once_with(coroutine) + + +def test_factory_exposes_interpreter_tools_on_path() -> None: + """vLLM subprocess helpers can find executables installed beside Python.""" + factory = load_factory_module() + + with ( + mock.patch.object(factory.sys, "prefix", "/workspace/.venv"), + mock.patch.dict(os.environ, {"PATH": "/usr/bin"}), + ): + factory.expose_interpreter_tools() + + assert os.environ["PATH"] == f"/workspace/.venv/bin{os.pathsep}/usr/bin" + + +def test_factory_avoids_flashinfer_jit_without_overriding_operator_choice() -> None: + """The wheel-only runtime does not require a host CUDA compiler by default.""" + factory = load_factory_module() + + with mock.patch.dict(os.environ, {}, clear=True): + factory.prepare_runtime_environment() + assert os.environ["VLLM_USE_FLASHINFER_SAMPLER"] == "0" + + with mock.patch.dict(os.environ, {"VLLM_USE_FLASHINFER_SAMPLER": "1"}, clear=True): + factory.prepare_runtime_environment() + assert os.environ["VLLM_USE_FLASHINFER_SAMPLER"] == "1" diff --git a/tools/inference_service_compiler/cli.py b/tools/inference_service_compiler/cli.py index 257aca49..08a15abd 100644 --- a/tools/inference_service_compiler/cli.py +++ b/tools/inference_service_compiler/cli.py @@ -15,7 +15,8 @@ from pydantic import BaseModel, ValidationError from inference_service_compiler.compiler import CompilationError, compile_intent, load_plan -from inference_service_compiler.models import InferenceIntent, LaunchReceipt, SecretEnvironmentVariable +from inference_service_compiler.models import LaunchReceipt, SecretEnvironmentVariable +from inference_service_compiler.profiles import load_profile from inference_service_compiler.runtime import ( RuntimeEffectError, cancel_run, @@ -26,7 +27,7 @@ probe_endpoint, ) -app = cyclopts.App(help="Compile and manage local inference services from typed intent.") +app = cyclopts.App(help="Compile and manage local inference services from TOML profiles.") P = ParamSpec("P") R = TypeVar("R") @@ -50,12 +51,12 @@ def wrapped(*args: P.args, **kwargs: P.kwargs) -> R: @command_errors def compile_plan( *, - intent: Path, + profile: Path, source_revision: str, output: Path | None = None, ) -> None: - """Compile a v1 intent JSON document without performing runtime effects.""" - parsed = InferenceIntent.model_validate_json(intent.read_text(encoding="utf-8")) + """Compile a v1 TOML profile without performing runtime effects.""" + parsed = load_profile(profile) plan = compile_intent(parsed, source_revision=source_revision) write_json(plan, output) diff --git a/tools/inference_service_compiler/compiler.py b/tools/inference_service_compiler/compiler.py index 9856c9b3..bfbbae59 100644 --- a/tools/inference_service_compiler/compiler.py +++ b/tools/inference_service_compiler/compiler.py @@ -201,18 +201,15 @@ def _compile_vllm( tuple[Capability, ...], tuple[CompatibilityEvidence, ...], ]: + if not isinstance(intent.task, Generation): + _raise_unsupported_task_engine(intent.task.kind, engine.kind) command, runtime = _vllm_command(intent, engine) declared = ("chat-completions",) - outcome = "characterized" if isinstance(intent.task, Generation) else "runtime-probe-required" evidence = ( CompatibilityEvidence( rule="vllm-openai-compatible-v1", - outcome=outcome, - detail=( - "vLLM exposes chat completions for generation" - if outcome == "characterized" - else "entity-detection capabilities must be established by the run probe" - ), + outcome="characterized", + detail="vLLM exposes chat completions for generation", ), ) return command, runtime, declared, evidence @@ -226,8 +223,8 @@ def _vllm_command( match intent.placement: case LocalProcessPlacement() as placement: argv = _literal_arguments( - engine.executable, - "serve", + engine.python_executable, + "tools/inference_service_compiler/vllm_server.py", intent.model.model_id, "--host", placement.host, diff --git a/tools/inference_service_compiler/models.py b/tools/inference_service_compiler/models.py index dab658dd..f75ee4de 100644 --- a/tools/inference_service_compiler/models.py +++ b/tools/inference_service_compiler/models.py @@ -94,7 +94,7 @@ class VllmEngine(FrozenModel): """vLLM's OpenAI-compatible server with bounded common options.""" kind: Literal["vllm"] = "vllm" - executable: str = Field(default="vllm", min_length=1) + python_executable: str = Field(default=".venv/bin/python", min_length=1) served_model_name: str | None = Field(default=None, min_length=1) api_key_env: str | None = Field(default=None, min_length=1) tensor_parallel_size: int | None = Field(default=None, ge=1) diff --git a/tools/inference_service_compiler/native_gliner.py b/tools/inference_service_compiler/native_gliner.py index 4dbdcf13..142a7cb9 100755 --- a/tools/inference_service_compiler/native_gliner.py +++ b/tools/inference_service_compiler/native_gliner.py @@ -36,10 +36,11 @@ from enum import StrEnum from typing import Protocol, cast -import structlog # type: ignore[unresolved-import] +import structlog # ty: ignore[unresolved-import] -- optional PEP 723 server dependency import uvicorn from cyclopts import App -from fastapi import FastAPI, HTTPException, Request # type: ignore[unresolved-import] + +_fastapi = importlib.import_module("fastapi") DEFAULT_HOST = "127.0.0.1" DEFAULT_PORT = 8001 @@ -133,23 +134,41 @@ def infer(self, chunks: list[str], params: DetectParams) -> list[list[Entity]]: """Detect normalized entities for each chunk.""" +class RequestLike(Protocol): + """Request surface consumed by the OpenAI-compatible endpoint.""" + + async def json(self) -> object: + """Decode the request body.""" + + +class NvidiaModel(Protocol): + """Local API consumed from the optional original GLiNER package.""" + + def inference(self, **kwargs: object) -> object: + """Run one original GLiNER batch.""" + + +class Gliner2Model(Protocol): + """Local API consumed from the optional GLiNER2 package.""" + + def batch_extract_entities(self, chunks: list[str], labels: list[str], **kwargs: object) -> object: + """Run one GLiNER2 batch.""" + + class NvidiaGlinerRuntime: """Adapter for the original `gliner` local inference API.""" - def __init__(self, model: object) -> None: + def __init__(self, model: NvidiaModel) -> None: self._model = model def infer(self, chunks: list[str], params: DetectParams) -> list[list[Entity]]: - raw = cast( - object, - self._model.inference( # type: ignore[attr-defined] - texts=chunks, - labels=list(params.labels), - threshold=params.threshold, - flat_ner=params.flat_ner, - relations=[], - batch_size=params.inference_batch_size, - ), + raw = self._model.inference( + texts=chunks, + labels=list(params.labels), + threshold=params.threshold, + flat_ner=params.flat_ner, + relations=[], + batch_size=params.inference_batch_size, ) return normalize_nvidia_output(raw) @@ -157,20 +176,17 @@ def infer(self, chunks: list[str], params: DetectParams) -> list[list[Entity]]: class Gliner2Runtime: """Adapter for GLiNER2's local batch extraction API.""" - def __init__(self, model: object) -> None: + def __init__(self, model: Gliner2Model) -> None: self._model = model def infer(self, chunks: list[str], params: DetectParams) -> list[list[Entity]]: - raw = cast( - object, - self._model.batch_extract_entities( # type: ignore[attr-defined] - chunks, - list(params.labels), - threshold=params.threshold, - include_confidence=True, - include_spans=True, - batch_size=params.inference_batch_size, - ), + raw = self._model.batch_extract_entities( + chunks, + list(params.labels), + threshold=params.threshold, + include_confidence=True, + include_spans=True, + batch_size=params.inference_batch_size, ) return normalize_gliner2_output(raw) @@ -572,7 +588,7 @@ def configure_logging(log_format: LogFormat) -> None: @asynccontextmanager -async def lifespan(_api: FastAPI) -> AsyncIterator[None]: +async def lifespan(_api: object) -> AsyncIterator[None]: """Own the local runtime and its single inference worker for API lifetime.""" global runtime, detector if state is None: @@ -589,7 +605,7 @@ async def lifespan(_api: FastAPI) -> AsyncIterator[None]: await detector.stop() -api = FastAPI(lifespan=lifespan) +api = _fastapi.FastAPI(lifespan=lifespan) app = api @@ -601,16 +617,16 @@ def list_models() -> dict[str, object]: @api.post("/v1/chat/completions") -async def chat_completions(request: Request) -> dict[str, object]: +async def chat_completions(request: RequestLike) -> dict[str, object]: """Detect requested entity labels and return Anonymizer's JSON-string content.""" if detector is None: - raise HTTPException(status_code=503, detail="GLiNER model is not loaded") + raise _fastapi.HTTPException(status_code=503, detail="GLiNER model is not loaded") try: body = require_mapping(await request.json(), "request") params = parse_detect_params(body) text = extract_text(body.get("messages", [])) except ValueError as exc: - raise HTTPException(status_code=422, detail=str(exc)) from exc + raise _fastapi.HTTPException(status_code=422, detail=str(exc)) from exc entities = await detector.detect(text, params) content = json.dumps({"entities": [entity.as_dict() for entity in entities]}) return { diff --git a/tools/inference_service_compiler/profiles.py b/tools/inference_service_compiler/profiles.py new file mode 100644 index 00000000..f00bd4d9 --- /dev/null +++ b/tools/inference_service_compiler/profiles.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Human-authored profile transport for the inference service compiler.""" + +from __future__ import annotations + +import tomllib +from pathlib import Path + +from inference_service_compiler.models import InferenceIntent + + +def load_profile(path: Path) -> InferenceIntent: + """Load and validate one TOML inference-service profile.""" + payload = tomllib.loads(path.read_text(encoding="utf-8")) + return InferenceIntent.model_validate(payload) diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index 4e9b54db..9054ed09 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -152,7 +152,7 @@ def _probe_or_cleanup( secret_values: Mapping[str, str], ) -> CapabilityProbeReceipt: try: - probe = wait_for_readiness(plan, secret_values=secret_values) + probe = wait_for_readiness(plan, secret_values=secret_values, handle=handle) except RuntimeEffectError as exc: cleanup_complete = _cleanup_handle(handle, plan.intent.lifecycle.shutdown_timeout_seconds) raise RuntimeEffectError( @@ -290,11 +290,20 @@ def wait_for_readiness( plan: RunPlan, *, secret_values: Mapping[str, str] | None = None, + handle: LocalProcessHandle | DockerHandle | None = None, ) -> CapabilityProbeReceipt: """Poll the declared readiness contract until it passes or times out.""" deadline = time.monotonic() + plan.readiness.timeout_seconds last_error: RuntimeEffectError | None = None while time.monotonic() < deadline: + if handle is not None and not is_handle_running(handle): + log_hint = f"; inspect {handle.stderr_path}" if isinstance(handle, LocalProcessHandle) else "" + raise RuntimeEffectError( + RuntimeDiagnostic( + code="launch-exited", + message=f"managed service exited before readiness{log_hint}", + ) + ) try: receipt = probe_endpoint(plan, secret_values=secret_values) if receipt.passed: diff --git a/tools/inference_service_compiler/vllm_factory.py b/tools/inference_service_compiler/vllm_factory.py new file mode 100644 index 00000000..e0dfcd1e --- /dev/null +++ b/tools/inference_service_compiler/vllm_factory.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Programmatic construction of vLLM's OpenAI-compatible server.""" + +from __future__ import annotations + +import argparse +import importlib +import os +import sys +from argparse import Namespace +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class VllmServerParameters: + """Bounded settings accepted by the source-owned vLLM server process.""" + + model: str + host: str + port: int + revision: str | None = None + tokenizer_revision: str | None = None + served_model_name: str | None = None + tensor_parallel_size: int | None = None + gpu_memory_utilization: float | None = None + max_model_len: int | None = None + enforce_eager: bool = False + lora_module: str | None = None + + +def parse_server_parameters(argv: Sequence[str]) -> VllmServerParameters: + """Parse the compiler's bounded process contract without using vLLM's CLI.""" + parser = argparse.ArgumentParser(description="Anonymizer-managed vLLM OpenAI server") + parser.add_argument("model") + parser.add_argument("--host", required=True) + parser.add_argument("--port", required=True, type=int) + parser.add_argument("--revision") + parser.add_argument("--tokenizer-revision") + parser.add_argument("--served-model-name") + parser.add_argument("--tensor-parallel-size", type=int) + parser.add_argument("--gpu-memory-utilization", type=float) + parser.add_argument("--max-model-len", type=int) + parser.add_argument("--enforce-eager", action="store_true") + parser.add_argument("--enable-lora", action="store_true") + parser.add_argument("--lora-modules") + parsed = parser.parse_args(list(argv)) + if parsed.enable_lora != (parsed.lora_modules is not None): + parser.error("--enable-lora and --lora-modules must be used together") + return VllmServerParameters( + model=parsed.model, + host=parsed.host, + port=parsed.port, + revision=parsed.revision, + tokenizer_revision=parsed.tokenizer_revision, + served_model_name=parsed.served_model_name, + tensor_parallel_size=parsed.tensor_parallel_size, + gpu_memory_utilization=parsed.gpu_memory_utilization, + max_model_len=parsed.max_model_len, + enforce_eager=parsed.enforce_eager, + lora_module=parsed.lora_modules, + ) + + +def build_server_arguments(parameters: VllmServerParameters) -> Namespace: + """Construct vLLM frontend and async-engine configs through its Python API.""" + arg_utils = importlib.import_module("vllm.engine.arg_utils") + cli_args = importlib.import_module("vllm.entrypoints.openai.cli_args") + model_protocol = importlib.import_module("vllm.entrypoints.openai.models.protocol") + + lora_modules = None + if parameters.lora_module is not None: + name, separator, path = parameters.lora_module.partition("=") + if not separator or not name or not path: + raise ValueError("LoRA module must use the form NAME=PATH") + lora_modules = [model_protocol.LoRAModulePath(name=name, path=path)] + + engine = arg_utils.AsyncEngineArgs( + model=parameters.model, + revision=parameters.revision, + tokenizer_revision=parameters.tokenizer_revision, + served_model_name=[parameters.served_model_name] if parameters.served_model_name is not None else None, + tensor_parallel_size=parameters.tensor_parallel_size or 1, + gpu_memory_utilization=parameters.gpu_memory_utilization or 0.9, + enforce_eager=parameters.enforce_eager, + enable_lora=lora_modules is not None, + ) + if parameters.max_model_len is not None: + engine.max_model_len = parameters.max_model_len + frontend = cli_args.FrontendArgs( + host=parameters.host, + port=parameters.port, + lora_modules=lora_modules, + ) + values = vars(engine) | vars(frontend) + values.update( + model_tag=None, + headless=False, + api_server_count=1, + config=None, + grpc=False, + ) + arguments = Namespace(**values) + cli_args.validate_parsed_serve_args(arguments) + return arguments + + +def expose_interpreter_tools() -> None: + """Expose console tools installed beside the selected Python interpreter.""" + interpreter_bin = str(Path(sys.prefix) / "bin") + current_path = os.environ.get("PATH", "") + path_entries = [entry for entry in current_path.split(os.pathsep) if entry != interpreter_bin] + os.environ["PATH"] = os.pathsep.join([interpreter_bin, *path_entries]) + + +def prepare_runtime_environment() -> None: + """Prepare a wheel-only vLLM runtime without requiring a host CUDA compiler.""" + expose_interpreter_tools() + os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0") + + +def run_server(argv: Sequence[str]) -> None: + """Construct and run vLLM's Python-owned OpenAI server lifecycle.""" + prepare_runtime_environment() + uvloop = importlib.import_module("uvloop") + api_server = importlib.import_module("vllm.entrypoints.openai.api_server") + api_utils = importlib.import_module("vllm.entrypoints.serve.utils.api_utils") + + api_utils.cli_env_setup() + arguments = build_server_arguments(parse_server_parameters(argv)) + uvloop.run(api_server.run_server(arguments)) diff --git a/tools/inference_service_compiler/vllm_server.py b/tools/inference_service_compiler/vllm_server.py new file mode 100644 index 00000000..975c391f --- /dev/null +++ b/tools/inference_service_compiler/vllm_server.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Internal process entry point for the programmatic vLLM server factory.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +TOOLS_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(TOOLS_ROOT)) + +from inference_service_compiler.vllm_factory import run_server # noqa: E402 + +if __name__ == "__main__": + run_server(sys.argv[1:]) diff --git a/tools/inference_service_profiles/gliner2.toml b/tools/inference_service_profiles/gliner2.toml new file mode 100644 index 00000000..95960786 --- /dev/null +++ b/tools/inference_service_profiles/gliner2.toml @@ -0,0 +1,30 @@ +schema_version = "inference-service.intent/v1" + +[task] +kind = "entity-detection" +dynamic_labels = true +offsets = true +scores = true + +[model] +kind = "hugging-face" +model_id = "fastino/gliner2-privacy-filter-PII-multi" +revision = "59894c087cb2923b01f337d4ee72f6ff84d5bdd6" + +[engine] +kind = "native-gliner" +family = "gliner2" +device = "auto" + +[placement] +kind = "local-process" +host = "127.0.0.1" +port = 8002 + +[access] +kind = "direct" + +[lifecycle] +kind = "managed" +startup_timeout_seconds = 300 +shutdown_timeout_seconds = 30 diff --git a/tools/inference_service_profiles/nvidia-gliner.toml b/tools/inference_service_profiles/nvidia-gliner.toml new file mode 100644 index 00000000..12262f7a --- /dev/null +++ b/tools/inference_service_profiles/nvidia-gliner.toml @@ -0,0 +1,30 @@ +schema_version = "inference-service.intent/v1" + +[task] +kind = "entity-detection" +dynamic_labels = true +offsets = true +scores = true + +[model] +kind = "hugging-face" +model_id = "nvidia/gliner-pii" +revision = "bd23e8ef4425fd04e34c5204ab49ffaa706eae79" + +[engine] +kind = "native-gliner" +family = "nvidia-gliner" +device = "auto" + +[placement] +kind = "local-process" +host = "127.0.0.1" +port = 8001 + +[access] +kind = "direct" + +[lifecycle] +kind = "managed" +startup_timeout_seconds = 300 +shutdown_timeout_seconds = 30 diff --git a/tools/inference_service_profiles/vllm-local.toml b/tools/inference_service_profiles/vllm-local.toml new file mode 100644 index 00000000..06d5399f --- /dev/null +++ b/tools/inference_service_profiles/vllm-local.toml @@ -0,0 +1,30 @@ +schema_version = "inference-service.intent/v1" + +[task] +kind = "generation" +chat = true + +[model] +kind = "hugging-face" +model_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" +revision = "fe8a4ea1ffedaf415f4da2f062534de366a451e6" + +[engine] +kind = "vllm" +python_executable = ".venv/bin/python" +served_model_name = "anonymizer-local" +gpu_memory_utilization = 0.85 +max_model_len = 2048 + +[placement] +kind = "local-process" +host = "127.0.0.1" +port = 8000 + +[access] +kind = "direct" + +[lifecycle] +kind = "managed" +startup_timeout_seconds = 600 +shutdown_timeout_seconds = 30 diff --git a/uv.lock b/uv.lock index deede134..8e87c400 100644 --- a/uv.lock +++ b/uv.lock @@ -205,25 +205,25 @@ wheels = [ [[package]] name = "apache-tvm-ffi" -version = "0.1.9" +version = "0.1.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6f/60/1e787a0b5ebf318483235be2a689ee367173983067e441b8379564f667c0/apache_tvm_ffi-0.1.9.tar.gz", hash = "sha256:d2d402587e8906de0a07f4746aa78f3d452c7efe3625d4bb39ac2ad693bce530", size = 2513731, upload-time = "2026-02-27T19:28:06.602Z" } +sdist = { url = "https://files.pythonhosted.org/packages/17/b0/5114e30faffe3279a51a5f3b45dd1b7ce09af1246b62447b45a39a374e54/apache_tvm_ffi-0.1.10.tar.gz", hash = "sha256:974c208766c304c780c17c6d405449e862f83b22c7b6b2b8c28b29d55a806ae3", size = 2691605, upload-time = "2026-04-07T19:58:51.767Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/42/b1/9f2cfd6d49b03c5d4ec5c12548d911e2e01265be783f343103b4df716765/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c0449fc3802987c3652bea266ffda2934a6f69c80bba791a3f55b91040656a18", size = 2231154, upload-time = "2026-02-27T19:27:15.691Z" }, - { url = "https://files.pythonhosted.org/packages/55/43/63faedea83494e99122466a993bcdccd31cf93c7e8a0d56731120e82e2b9/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6f16d73a82a9e68a439b7d233d48b1b929be17fe92df4bbf1ee2274e573144a3", size = 2323130, upload-time = "2026-02-27T19:27:17.259Z" }, - { url = "https://files.pythonhosted.org/packages/27/96/d735bc4c528efaf0a8a954076963c727aad2dde8577641aa9025ec4f2d52/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01ebb1308b2666c206aa9a4015eb48f03a5d98ea2e9cfb002bd5e2ca0b9c7ef3", size = 2159854, upload-time = "2026-02-27T19:27:18.789Z" }, - { url = "https://files.pythonhosted.org/packages/e4/3b/6cfc82a3ab5d9e501bbcee5df36eebe09da1c384461d7a55e2a17776d117/apache_tvm_ffi-0.1.9-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21365abd2a2a1a6d3b4e6e4f048309651125becfa795440c3607f3cc27d30ac7", size = 2307140, upload-time = "2026-02-27T19:27:20.222Z" }, - { url = "https://files.pythonhosted.org/packages/a7/c0/6d3d54f50012255b41bc3e24944c086f63c4707c8686c7c6780e9283eb96/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d503029e66c43b1a1cb1a42a1e9bb428c8a28dcbdec31c28e705472ca648a3a", size = 2203712, upload-time = "2026-02-27T19:27:25.867Z" }, - { url = "https://files.pythonhosted.org/packages/c6/dd/2bab4c6cd86257dbf99e93452a1af833113f8dc3e25a25579f6e4e4c8a94/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28241371934ea8af10d5067087ba1229ebddded7b2c02d33a258ec2a96df8c46", size = 2299704, upload-time = "2026-02-27T19:27:27.477Z" }, - { url = "https://files.pythonhosted.org/packages/7a/4a/b469bcb2e1014cb84d336d2a59f42958a058251c577a4c2680cacad346e2/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:87cacce81df55685fc6a76e1e3c5db1200e85e87bf5974b692c59d131b7bc622", size = 2130865, upload-time = "2026-02-27T19:27:29.092Z" }, - { url = "https://files.pythonhosted.org/packages/70/ef/5402da5d37f5270fd88ea0348acca78dba9be8bdbf6c2bcae0935eb03ef1/apache_tvm_ffi-0.1.9-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f45eb43499acac45ff6c93564f0ff2d3ca27b69656d540fd56ce59d51c0b4c65", size = 2278991, upload-time = "2026-02-27T19:27:30.729Z" }, - { url = "https://files.pythonhosted.org/packages/b4/a7/1e0643949e683fb3cfababd87058c0cfef122d1a3bb6ce703f719051b842/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d1f4d2b7ec7b1213632e9a104e9330bfc3dec48decffa62114c33aa188c9f43a", size = 2215954, upload-time = "2026-02-27T19:27:35.872Z" }, - { url = "https://files.pythonhosted.org/packages/d6/06/5016191ab61d2db4c3a7d754a3c1184e0836f575a7d08491669738c5e4b9/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4f01d16ba53fe118e363f7257253f07003797e4abe6fc9567f23b6a930dbff2", size = 2307291, upload-time = "2026-02-27T19:27:37.527Z" }, - { url = "https://files.pythonhosted.org/packages/e3/f5/40bf0667330938efbfc0a51743cc53c79e41b4ece1a8abad3076192c9674/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c0581dd6bfbce7b017ef85cfda08bbe38891cc4b3afbcfaa8bc2d383728e426", size = 2143850, upload-time = "2026-02-27T19:27:40.437Z" }, - { url = "https://files.pythonhosted.org/packages/72/4a/421cbd4ed32e8bad3b88af3e8fa145c1f6f493bdd05be15b6f2d9b3cb7d6/apache_tvm_ffi-0.1.9-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dfa14be2a49347791ef21222a8225ce7f99bfec17104a676cb4f1bf3a107088", size = 2289038, upload-time = "2026-02-27T19:27:41.972Z" }, + { url = "https://files.pythonhosted.org/packages/55/c3/598da8bf49e850aa329a024929643eb141d7907f4d97705b74e49ca499f6/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d5cf055a83e1b1944dd05386c593bc22de29a1aeb6cae45af54735796875194a", size = 2543849, upload-time = "2026-04-07T19:58:05.419Z" }, + { url = "https://files.pythonhosted.org/packages/50/58/221b41c5f77405f99875754f2a38c01da49387e366bf0fd40302b2cd25f3/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:81c4144fc06750312f2829960862bd52ba6f0bb17e6d7aae3f7a09f9170f7e7a", size = 2650260, upload-time = "2026-04-07T19:58:07.002Z" }, + { url = "https://files.pythonhosted.org/packages/01/2b/36b5210d24492dc4dda488d785dd4039c0788238f6aa4aa5067b2ea494d1/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7bafe9a6191c77f3978e9cd9726799abbe7fd574913fa2416402bc876633524e", size = 2459987, upload-time = "2026-04-07T19:58:08.409Z" }, + { url = "https://files.pythonhosted.org/packages/9f/36/8f8f719c1c52ed978fc99acde51827f5fc48380e69a310a02a6a5ae94d0f/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2ba653825f806a87fe2ca48ebab1abb9ae0f17d6642fbada622c6c5eea9fe96", size = 2631364, upload-time = "2026-04-07T19:58:09.784Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2a/1978a1c827e1212de4f369ec08cfeb44719bbe6cbeab90b15e967c68c108/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ec5c4a81e294e6379e4dea68c86266924d3f22829c3de272806c980238e43e59", size = 2476596, upload-time = "2026-04-07T19:58:14.316Z" }, + { url = "https://files.pythonhosted.org/packages/50/6f/23740f06829030704e6f8f1f7093a06b7a68f904baa40053a5f594705bae/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:73d478395a8625dd92fde7b7fd92b4719f18f480b78336e422cb66cc7985213d", size = 2589574, upload-time = "2026-04-07T19:58:15.94Z" }, + { url = "https://files.pythonhosted.org/packages/92/d0/54badf5c8f6208e06f331a20ddd154f19c94c2e906da5b8cce7d60727d4b/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3829216a8500c2f61062e48c627f6db6c3fa49416b3ffa85bc04243ae5d759f7", size = 2396434, upload-time = "2026-04-07T19:58:17.519Z" }, + { url = "https://files.pythonhosted.org/packages/51/f7/ca3fdadc2468e8b67a2f3f13bb7aa132c584feefd8a25dbf920e4bf0a03b/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:96b69030c722572e13e30182733adfa2d604258e988b3f6630a16f397c7f9288", size = 2571084, upload-time = "2026-04-07T19:58:20.399Z" }, + { url = "https://files.pythonhosted.org/packages/2e/5d/b1661512164772fc9ef1642234bf117182b440fc0a0b2ca8bd829fe7b40e/apache_tvm_ffi-0.1.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32b9f4a44c09fcdd0994ee3c4415bf0371d68ea35a46da94ddcc666c9a6cf677", size = 2508518, upload-time = "2026-04-07T19:58:25.3Z" }, + { url = "https://files.pythonhosted.org/packages/d2/57/7266807b34344b9d8e4d776ebff38fd25f93a73e8c24bc595a67b6b69b3c/apache_tvm_ffi-0.1.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c9b93dc7fdc99d4cc44e9ac95063073b4fb8ced94929197ea3d631b70f554d8a", size = 2617108, upload-time = "2026-04-07T19:58:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/96/c3/a152ed68f57a491baaf70819224b98643309c7488fdcbc6fa3c84ebb9ca8/apache_tvm_ffi-0.1.10-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74724db54dfb825951e2deb3d2024b2c1867bff456db81512e475f9ccdd9b86b", size = 2432434, upload-time = "2026-04-07T19:58:28.681Z" }, + { url = "https://files.pythonhosted.org/packages/c4/09/5e2877c635edc8ac83caa106a6e78bd4816cbc2e52e1daea652c1fe956cf/apache_tvm_ffi-0.1.10-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ac03c04145d9c248992e6f2ec2392a6914966a416eeeeaa729393f40b047be42", size = 2602517, upload-time = "2026-04-07T19:58:30.35Z" }, ] [[package]] @@ -696,7 +696,7 @@ wheels = [ [[package]] name = "compressed-tensors" -version = "0.15.0.1" +version = "0.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "loguru" }, @@ -704,9 +704,9 @@ dependencies = [ { name = "torch" }, { name = "transformers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/41/1b/c3c4a98ec5f2727656336f07a0c35862195c310d8eb0b2fa5b4be6848680/compressed_tensors-0.15.0.1.tar.gz", hash = "sha256:a8e93054e8a5ec49c980b09ed36c4c1249b4a8ee167920a8e461c4da26e78d99", size = 229412, upload-time = "2026-04-10T14:23:54.708Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2c/9e/d7f18bd9a0354088abc11a0c1f2c7698f7c49e5a709faedf6a46e388f693/compressed_tensors-0.17.0.tar.gz", hash = "sha256:15c20d06bdbcf35b51fc99fd125e7b9be1e1855567c33b7a46dfac26ad6fb126", size = 257091, upload-time = "2026-06-03T16:49:17.208Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/52/93833dc1610e017ac5b7dcd59b8304d8ef67d1114c2d124e728a2cbbea12/compressed_tensors-0.15.0.1-py3-none-any.whl", hash = "sha256:e1b1f322e82e475715e242bad46925a304ea8e5c98b5055a15b8eb22fb6bfea9", size = 194260, upload-time = "2026-04-10T14:23:53.098Z" }, + { url = "https://files.pythonhosted.org/packages/35/63/6edf0415b072fff0bf8b546074dea3f0f9b148e49b601ac98bdc60a76c68/compressed_tensors-0.17.0-py3-none-any.whl", hash = "sha256:4a1b89b508f7efb8ffb4eee8a6e69e0452d9b080cae130146025c64fbe9fa9aa", size = 211714, upload-time = "2026-06-03T16:49:15.672Z" }, ] [[package]] @@ -1220,15 +1220,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c9/7a/cef76fd8438a42f96db64ddaa85280485a9c395e7df3db8158cfec1eee34/dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7", size = 116252, upload-time = "2024-01-27T23:42:14.239Z" }, ] -[[package]] -name = "diskcache" -version = "5.6.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, -] - [[package]] name = "distlib" version = "0.4.0" @@ -1355,7 +1346,7 @@ wheels = [ [[package]] name = "fastapi" -version = "0.141.1" +version = "0.136.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-doc" }, @@ -1364,9 +1355,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +sdist = { url = "https://files.pythonhosted.org/packages/81/2d/ff8d91d7b564d464629a0fd50a4489c97fcb836ac230bf3a7269232a9b1f/fastapi-0.136.3.tar.gz", hash = "sha256:e487fae93ad408e6f47641ee4dfe389864fd7bec92e547ea8498fc13f43e83ab", size = 396410, upload-time = "2026-05-23T18:53:15.192Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, + { url = "https://files.pythonhosted.org/packages/e0/82/45359b62a067409bd929ae8a56b8ed13e5a8c8a61194b3c236920999ab83/fastapi-0.136.3-py3-none-any.whl", hash = "sha256:3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620", size = 117481, upload-time = "2026-05-23T18:53:16.924Z" }, ] [package.optional-dependencies] @@ -1533,17 +1524,9 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, ] -[[package]] -name = "flashinfer-cubin" -version = "0.6.8.post1" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/11/b7/5e3b1a8c67031b421a8bd29c2bc29b900a550bb3392e8bda18bb15b5e476/flashinfer_cubin-0.6.8.post1-py3-none-any.whl", hash = "sha256:43636d4cd39e694a83d76a89f87fefcdf4cecb4c4f7dd22dac25ec368c1e901f", size = 295154113, upload-time = "2026-04-18T18:28:21.738Z" }, -] - [[package]] name = "flashinfer-python" -version = "0.6.8.post1" +version = "0.6.14" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-tvm-ffi" }, @@ -1562,9 +1545,9 @@ dependencies = [ { name = "torch" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/53/1e/2760fef9e74abc4480961048e5790b4c9e955872fb4d7d97900cfddced5a/flashinfer_python-0.6.8.post1.tar.gz", hash = "sha256:b18e4121baf9b93fa9a9f368ba9b981a0342895f50ab9dddc224aeb964ed346f", size = 6675885, upload-time = "2026-04-18T18:28:13.299Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8f/11/ce2271271bee6990d34ed2d01288e9e92a0ea8ee45fb28de8e746c7da761/flashinfer_python-0.6.14.tar.gz", hash = "sha256:f4da8b5e005601784e85e0dcaa3389f908ee2d32c2560142d67124ab10e4a070", size = 9944949, upload-time = "2026-07-02T00:22:50.879Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/73/6d/1e8a8533913e33a50a486332ce0673f4fdb860f6eb9ed450327c5c1762cb/flashinfer_python-0.6.8.post1-py3-none-any.whl", hash = "sha256:818f9b8cc2fe66c42a1f6264be4841ac8821ada703685a02cfccb2b5124a710b", size = 9385316, upload-time = "2026-04-18T18:28:10.285Z" }, + { url = "https://files.pythonhosted.org/packages/f2/8f/b101913cb2b3687654f56681cfe9836d447526be663c149966470ef70531/flashinfer_python-0.6.14-py3-none-any.whl", hash = "sha256:d124369346a3d48eac67e31c42f7a3c813bcc0abc10e2e36db413b7b3dfd97df", size = 14574383, upload-time = "2026-07-02T00:22:48.413Z" }, ] [[package]] @@ -1695,22 +1678,6 @@ http = [ { name = "aiohttp" }, ] -[[package]] -name = "gguf" -version = "0.19.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, - { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/ae/17f1308ae45cd7b08ebb521747d5b23f4efc4d172038a4e228dd5106c3ff/gguf-0.19.0.tar.gz", hash = "sha256:dbadcd6cc7ccd44256f2229fe7c2dff5e8aa5cf0612ab987fd2b1a57e428923f", size = 111220, upload-time = "2026-05-06T13:04:03.667Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/bb/d71d6da82763528c2c2ed6b59a9d6142c6595545a4c448e2085d155e88c2/gguf-0.19.0-py3-none-any.whl", hash = "sha256:70bcd10edfe697fb2dad6e40af2234b9d8ece9a41a99761405121ebda1c3c1cd", size = 118475, upload-time = "2026-05-06T13:04:02.588Z" }, -] - [[package]] name = "ghp-import" version = "2.1.0" @@ -1978,6 +1945,36 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/de/d8/95b735e183957c1f26d94c52977f09d466d55119cbbc1558ea4975e4c216/huggingface_hub-1.27.0-py3-none-any.whl", hash = "sha256:7df6827c2f956c60fbaa64646e979e566db76f619dd0a9729dfb8c5a3eb4f68d", size = 784926, upload-time = "2026-08-07T12:48:02.905Z" }, ] +[[package]] +name = "humming-kernels" +version = "0.1.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings" }, + { name = "jinja2" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "nvidia-ml-py" }, + { name = "pyelftools" }, + { name = "safetensors" }, + { name = "tabulate" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "triton" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/4f/6977a31451c3f7aa1deaa76506d6cdeb74ef418ad8bcba2e98f7510b26ec/humming_kernels-0.1.10.tar.gz", hash = "sha256:da3e46fb9fc9eba2a9327c2e8135ead68e390c955acd7449f97ee7c71666c8b1", size = 220110, upload-time = "2026-07-02T10:22:57.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/ba/869bc24591d2b4fb0d8da821528072971052a934af077a11f77a0f2b3e79/humming_kernels-0.1.10-py3-none-any.whl", hash = "sha256:4ded0998ff085afeddde70baf93f97c2929969ec3d4a63a52cfec5072bc972b4", size = 184889, upload-time = "2026-07-02T10:22:56.031Z" }, +] + +[package.optional-dependencies] +cu13 = [ + { name = "nvidia-cuda-cccl" }, + { name = "nvidia-cuda-nvcc" }, + { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-runtime" }, +] + [[package]] name = "identify" version = "2.6.16" @@ -2607,12 +2604,14 @@ wheels = [ [[package]] name = "llguidance" -version = "1.3.0" +version = "1.7.6" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/95/48/3f7a9d3ff1b36bba92b5107a3a21286821227afe9ea464736133994d61fb/llguidance-1.3.0.tar.gz", hash = "sha256:861249afd51dc325646834462ea827e57a5c2b2042e108e6aae7059fdad9104d", size = 1070460, upload-time = "2025-10-20T19:58:44.164Z" } +sdist = { url = "https://files.pythonhosted.org/packages/da/91/6bc8bb503dc259e46d253b5424385a54fe06c38a4c7a12befe69a3c2455a/llguidance-1.7.6.tar.gz", hash = "sha256:db7febbe412ed2015501904646750071d7e00e6df7f85c4b956ad4f206fd2df7", size = 1156574, upload-time = "2026-06-03T20:13:25.316Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/aa/11/44389d3d1526d7a5c38ffd587a5ebc61d7bee443ac1dea95f2089ad58f5f/llguidance-1.3.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6f6caca5d78db7f76e1fbb0fff8607b861c32d47fa3d5dee2fc49de27ee269df", size = 2835242, upload-time = "2025-10-20T19:58:34.518Z" }, - { url = "https://files.pythonhosted.org/packages/83/a8/1ff2bedb8f9acb46a2d2d603415d272bb622c142ea86f5b95445cc6e366c/llguidance-1.3.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc17e9dd602c3879bf91664a64bf72f54c74dbfbeb24ccfab6a5fe435b12f7aa", size = 3033133, upload-time = "2025-10-20T19:58:38.721Z" }, + { url = "https://files.pythonhosted.org/packages/67/da/28756068fa9f7147874fcd712e7317c24785f25d762a96e901850d9a2f5f/llguidance-1.7.6-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0444020249cde1292f13acf786e35c245fd3572d466877d2734824a9026e55aa", size = 3470362, upload-time = "2026-06-03T20:12:59.813Z" }, + { url = "https://files.pythonhosted.org/packages/11/90/37cc12dd44c1f8fd84d5cc4e293467febe5a9899d6b55805485af7c21c9a/llguidance-1.7.6-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e4f2a489c1c3943bb1b3c206b45794153cb6954f45cd3de8e02198319ddc6b1", size = 3485304, upload-time = "2026-06-03T20:13:03.504Z" }, + { url = "https://files.pythonhosted.org/packages/51/b9/dc76d7716e04dc7b3427cae52eaa32bd20771382d4d1dd9f4538a9dd2086/llguidance-1.7.6-cp39-abi3-manylinux_2_31_aarch64.whl", hash = "sha256:e70fa25ed550c2b50c2fd70baa9e2808b4ecb859d01e453bd5459aff62ba38c3", size = 2899993, upload-time = "2026-06-03T20:13:13.563Z" }, + { url = "https://files.pythonhosted.org/packages/1a/64/d74336f22242ef94356a456057d4ff1be7c1bc9c7dbc867171c6982a5512/llguidance-1.7.6-cp39-abi3-manylinux_2_31_x86_64.whl", hash = "sha256:ceec951d29a74309984e3be0fe7f5f56c1362434cd937abd517b259a60908b1e", size = 3074809, upload-time = "2026-06-03T20:13:15.498Z" }, ] [[package]] @@ -3491,7 +3490,7 @@ docs = [ { name = "mkdocs-material" }, { name = "mkdocstrings", extras = ["python"] }, ] -local-models = [{ name = "vllm", marker = "sys_platform == 'linux'", specifier = "==0.20.0" }] +local-models = [{ name = "vllm", marker = "sys_platform == 'linux'", specifier = "==0.26.0" }] measurement = [{ name = "wandb", extras = ["workspaces"], specifier = ">=0.19,<1" }] notebooks = [ { name = "datasets", specifier = ">=4.0.0,<6" }, @@ -3782,6 +3781,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236, upload-time = "2025-10-09T08:59:32.536Z" }, ] +[[package]] +name = "nvidia-cuda-cccl" +version = "13.3.3.4.1" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/96/bd/572971ffc14bd36676c821fc15d991b08fe6179cb09368250147475f954d/nvidia_cuda_cccl-13.3.3.4.1-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:067d19b4b3c9d0f2ebec9f29a311b2863db96bf98e058bbc331597d51ce818cf", size = 3454030, upload-time = "2026-06-29T16:41:49.092Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ab/049726d90147865a3ea53bae6cb7c35b98bf1fdf96cdb967101329625f83/nvidia_cuda_cccl-13.3.3.4.1-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc0adc188d570b09f4d606c7dc05a42aa3d8aa082e0d60f7bbfc5b6435f627c6", size = 3454034, upload-time = "2026-06-29T16:42:07.435Z" }, +] + +[[package]] +name = "nvidia-cuda-crt" +version = "13.3.73" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/41/2089e411507d66458d67208bdd1bc562d492bb6458c3d2aea4603072a219/nvidia_cuda_crt-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:60aacc0b5e1e8b40c62abe4d1ab16440add91b99bd2f17f62dd091586b73d166", size = 157353, upload-time = "2026-06-29T16:42:38.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ce/16d76f4b5b3f7460f5ebd17516685495c149c66651ffdd381f90e4d4e65c/nvidia_cuda_crt-13.3.73-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df14a17ae1c5c3171265411212246654d780f89344ea85344466c6b955247543", size = 157352, upload-time = "2026-06-29T16:43:09.209Z" }, +] + [[package]] name = "nvidia-cuda-cupti" version = "13.0.85" @@ -3791,6 +3808,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, ] +[[package]] +name = "nvidia-cuda-nvcc" +version = "13.3.73" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cuda-crt" }, + { name = "nvidia-cuda-runtime" }, + { name = "nvidia-nvvm" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/14/9f5cdc994d5431e2f08f62ffe34509e7feabd1f2e18517e2d7720c6ff0fd/nvidia_cuda_nvcc-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:70f250825355d2c3aa6c7a972a0ec00f020bad66d2679e527eb4336301c904aa", size = 39515578, upload-time = "2026-06-29T16:47:40.318Z" }, + { url = "https://files.pythonhosted.org/packages/83/19/e46ef3597ba47a9f8a91ab24533db42a600b659fc418dbe4af0b630bcb41/nvidia_cuda_nvcc-13.3.73-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f483af83166c4fa356a21606076d553b0b4ceaebbd9912e537545080db695bdd", size = 44942138, upload-time = "2026-06-29T16:48:13.615Z" }, +] + [[package]] name = "nvidia-cuda-nvdisasm" version = "13.3.73" @@ -3832,17 +3863,19 @@ wheels = [ [[package]] name = "nvidia-cudnn-frontend" -version = "1.18.0" +version = "1.27.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/9a/83d3d080118de4a7810fa019349edec634b8b37b9cafaacd05719de62dd6/nvidia_cudnn_frontend-1.18.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6d4d0b88d617b233a503c84980b54d840b60b2734497d1a7a071ec5293daec2", size = 2023709, upload-time = "2026-01-27T23:32:10.912Z" }, - { url = "https://files.pythonhosted.org/packages/13/c7/c3624b3ed77b102618f26295e816b27f1c3ebb1143730237a9f51d403c3f/nvidia_cudnn_frontend-1.18.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:382ea063b92cbfd5b442cb75ff8422932d78276aecf139e46713ed1ad3d07af4", size = 2155568, upload-time = "2026-01-27T23:07:13.277Z" }, - { url = "https://files.pythonhosted.org/packages/e3/b4/604e230378680ee117849a4e1045baca092f93161a829291a84d5acce70c/nvidia_cudnn_frontend-1.18.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:310b417f2848a83d1437203fcaeea320a74fb7f28af20bf42bf5afc9c01f1c12", size = 2027408, upload-time = "2026-01-27T23:32:46.576Z" }, - { url = "https://files.pythonhosted.org/packages/c6/52/08f98262e77b1cbcc834cc1a5db494d0661ea1dbdea58c2e2d51a57fdaca/nvidia_cudnn_frontend-1.18.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c023539ca6de99234cf5102c3ec0d6af817f5396fc93028a22ba5b834a35b8a", size = 2159245, upload-time = "2026-01-27T23:07:32.664Z" }, - { url = "https://files.pythonhosted.org/packages/e8/bd/db791a26ebb6a6e1268f518e18c82d8ad18546f7008f4b0d5bde15f927de/nvidia_cudnn_frontend-1.18.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5a6e2b7bd43705ffa4af3b187374fdd5e7d09fc228a4d65fc8b4b0a537a8e605", size = 2027249, upload-time = "2026-01-27T23:33:22.46Z" }, - { url = "https://files.pythonhosted.org/packages/19/74/3038cf496d5de7cfdff730f5202e438c17d9123de507059340e02ddff9d7/nvidia_cudnn_frontend-1.18.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0544206b02cae9da4f044ca3fe7416b99e0c8a8052285dd3e5a8fc445d34f9c", size = 2160001, upload-time = "2026-01-27T23:07:50.248Z" }, - { url = "https://files.pythonhosted.org/packages/a3/0a/515209dd2afc6027bf1112bf415f575bfe9628d18877abe7424cb597dd7b/nvidia_cudnn_frontend-1.18.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b489da1b30f1d7da822b37b89cc4f68afd80e020eb57e4ab24921f8b57f6e946", size = 2028689, upload-time = "2026-02-11T21:32:04.235Z" }, - { url = "https://files.pythonhosted.org/packages/ab/57/52d18e1f50979eeabfafb408ec73068afc5a1e1ccd21636240317cd456d4/nvidia_cudnn_frontend-1.18.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:37688c81a34ac590aff9de4c34d2968bab949411af707baa327616ebd4b34ae1", size = 2160182, upload-time = "2026-02-11T21:25:18.437Z" }, + { url = "https://files.pythonhosted.org/packages/f7/8e/c185018bdfd1831aaadd1557270f0a0f5dc8a88c3e4062b69473dfea7956/nvidia_cudnn_frontend-1.27.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b881178e3dd3b4ad081be88600756e26585f3ae48c0b5fb687577721793b5ead", size = 4586031, upload-time = "2026-08-06T22:41:02.117Z" }, + { url = "https://files.pythonhosted.org/packages/83/f6/60e4b4b52af0af85b479fca058357ce64fde1c69acf32c01dd8cbb514859/nvidia_cudnn_frontend-1.27.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5cf124a0fbce9c49417122ba6de903858c766e3d8690d8809372d40a0a6c9256", size = 4746954, upload-time = "2026-08-06T22:41:26.027Z" }, + { url = "https://files.pythonhosted.org/packages/df/cd/d6b6910b79389955d9c33596c03380db63d76f1bcc6cdd24efc3ced68a3b/nvidia_cudnn_frontend-1.27.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:649b9f5a20bded17bc917122bcabd83826b82cb9bbd5b74573b769a9f4930798", size = 4589494, upload-time = "2026-08-06T22:42:10.993Z" }, + { url = "https://files.pythonhosted.org/packages/cb/b5/3a998fa2ba1aa527b35136d2c675ee3f8394c6a7f30605c63c3d9b64023b/nvidia_cudnn_frontend-1.27.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef1f1b4927f2f9e76a5ba83ac4bdc01a6a84750032429ea9c132599ec60c8947", size = 4749917, upload-time = "2026-08-06T22:42:36.857Z" }, + { url = "https://files.pythonhosted.org/packages/e1/d5/4956ea43565c91a66c71b13e0ce35b3a63171176193a96d6569d8c01c494/nvidia_cudnn_frontend-1.27.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa1caac464663202f8cc83fdeca02b0fd99cc43a538025758e6444ed16e312b5", size = 4589240, upload-time = "2026-08-06T22:43:19.396Z" }, + { url = "https://files.pythonhosted.org/packages/bf/cf/a6be6aa24e767aff2def7bf7797eb5bf341362f2f1746a48e8476ce6955e/nvidia_cudnn_frontend-1.27.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1da45d4de55394cce12a596776d6250cf8f98c14481d241444e881b1760adb83", size = 4749803, upload-time = "2026-08-06T22:43:39.494Z" }, + { url = "https://files.pythonhosted.org/packages/89/94/6cd1c9d8026530193684949a285649ec446d68d9c34c1a7cd3d3a9f0e0aa/nvidia_cudnn_frontend-1.27.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac516233e99f28e48d467c5d9f29b8af10eeed8531143458b858dab241d47114", size = 4591229, upload-time = "2026-08-06T22:44:23.34Z" }, + { url = "https://files.pythonhosted.org/packages/a5/1a/de71f045c0a81d97997080e1487ee26b0179fe67f411977b12c08cd33ecf/nvidia_cudnn_frontend-1.27.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8011f768284d6a115e2f6a4b62a30500f2b3fa4330ec54cf5c9e58719b04aaf", size = 4750787, upload-time = "2026-08-06T22:44:47.744Z" }, + { url = "https://files.pythonhosted.org/packages/ed/e0/85cabd232201611c8edc2711f93fc2f14b0e00619f819697d5385be1665f/nvidia_cudnn_frontend-1.27.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:835771ecd230fbd2e390a255255dd5989d179882cc7aecfc583877619bc7e8c0", size = 4597464, upload-time = "2026-08-06T22:45:33.521Z" }, + { url = "https://files.pythonhosted.org/packages/5f/8d/fda467c47ed849bb1605ac0c89ce9a5c3066779111dc969a3397b7693169/nvidia_cudnn_frontend-1.27.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bc5fdde1ffb2a88e9cfc85c997c7dfa2e557755edfa31d0607adc7a450b822d", size = 4752677, upload-time = "2026-08-06T22:45:55.192Z" }, ] [[package]] @@ -3912,19 +3945,24 @@ wheels = [ [[package]] name = "nvidia-cutlass-dsl" -version = "4.6.2" +version = "4.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nvidia-cutlass-dsl-libs-base" }, { name = "nvidia-cutlass-dsl-libs-cu12" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/b2/ec/14e6ecbfed31ec35bbd1bb6965ae61f879370906a8f9c2e851e292704ba4/nvidia_cutlass_dsl-4.6.2-py3-none-any.whl", hash = "sha256:06ac62deb182a852dfb053032cf945bf02b9aa1bf502a18b129d280d7babb26c", size = 10460, upload-time = "2026-08-05T14:39:41.657Z" }, + { url = "https://files.pythonhosted.org/packages/8b/1c/fbddb760a0228df87a9e9d1e60b76ecbe6e18035f5853efe0b4563651b2b/nvidia_cutlass_dsl-4.6.0-py3-none-any.whl", hash = "sha256:e3e0e4d8df20d82c8401fa013f4d82021f41daa5fca3d24b55d4a677f2308ca8", size = 10459, upload-time = "2026-07-02T03:23:18.43Z" }, +] + +[package.optional-dependencies] +cu13 = [ + { name = "nvidia-cutlass-dsl-libs-cu13" }, ] [[package]] name = "nvidia-cutlass-dsl-libs-base" -version = "4.6.2" +version = "4.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cuda-python" }, @@ -3936,21 +3974,21 @@ dependencies = [ { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/70/0e/5020da76c7dd3ed74acc6f435e53ca2df967762b2d8408f3e81f146e018a/nvidia_cutlass_dsl_libs_base-4.6.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:aae76fcc94c5483279f06848ee470b87417b9b1a4ad8f822a12e2654cbaf513a", size = 3321401, upload-time = "2026-08-05T14:11:32.779Z" }, - { url = "https://files.pythonhosted.org/packages/12/ce/dfefb22eb438de1e0e6c6627f188449ad3f65448522ba80a12a67bb14715/nvidia_cutlass_dsl_libs_base-4.6.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:551097a99537ffdd239678088865ab8d1e273633f0961493509284d8445dd420", size = 2827063, upload-time = "2026-08-05T14:11:54.706Z" }, - { url = "https://files.pythonhosted.org/packages/aa/f3/4be98f2faa283471e355a42ecbc20956fb2c3b6dbc71861f483be277e99a/nvidia_cutlass_dsl_libs_base-4.6.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e129db7155d56ba4478be098c0f819e37690a6ce5d43a9a9a83bf1300d32c57f", size = 3321894, upload-time = "2026-08-05T14:12:30.139Z" }, - { url = "https://files.pythonhosted.org/packages/54/1b/cf4a0837054bb4b9dc36cdd86f9d2e7fead3629e3373aaa7f7ec1e5e1542/nvidia_cutlass_dsl_libs_base-4.6.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:24dfaddad6077fd0de14eecaf66a72762f2f5f30ae1872231f5bf8b243ac2eac", size = 2824987, upload-time = "2026-08-05T14:12:49.618Z" }, - { url = "https://files.pythonhosted.org/packages/41/e9/a402e079e926f1fbcf82e30dfcd87aa924aaf91b02db08d57e83b158dbb2/nvidia_cutlass_dsl_libs_base-4.6.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:334a9be3c5054286666b14f81c9278075c2007b947ba75f7ee44b94aadf26415", size = 3321794, upload-time = "2026-08-05T14:13:12.501Z" }, - { url = "https://files.pythonhosted.org/packages/d8/4c/0a6c6e785e8fb2bfc6a0941559d30fbaf6264ae77ee1a26a42e37fd6efdc/nvidia_cutlass_dsl_libs_base-4.6.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:896f8a87d0815e59066b0607930c9905a5cf8f5c7a01a423b0093bc876bac4b4", size = 2825075, upload-time = "2026-08-05T14:13:34.746Z" }, - { url = "https://files.pythonhosted.org/packages/47/9d/627165c9044426610176c4966ca8fd0fb9856684db918e725f6561872f8a/nvidia_cutlass_dsl_libs_base-4.6.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:f35403515f5ba5bc69924f9faf0206af4bde3ebef9117d5f696529102cf4dbf7", size = 3321977, upload-time = "2026-08-05T14:14:01.683Z" }, - { url = "https://files.pythonhosted.org/packages/a8/b8/50b0fedea6db2a522f299652d6187b74de1beb74f201a1533c29cec9fb67/nvidia_cutlass_dsl_libs_base-4.6.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:17a0165624144a2e6962d6e2322cd9008050326808b89a649877450e732403e4", size = 2825099, upload-time = "2026-08-05T14:14:22.171Z" }, - { url = "https://files.pythonhosted.org/packages/d9/91/a904783c7032da5c56a1e24f5daba5407018a356668798a71dbd69fc726a/nvidia_cutlass_dsl_libs_base-4.6.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:77266dc557bc2575244a19b1c43fc971252cf152b198cd872553c2734e6a4e44", size = 3330162, upload-time = "2026-08-05T14:14:57.698Z" }, - { url = "https://files.pythonhosted.org/packages/4b/bf/ce685d267cab2728ccacf7d47c8a52ab600fab6d67865d5be7e1e971bb8e/nvidia_cutlass_dsl_libs_base-4.6.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:b8fd31f484d6231ea11c29cee2aebeaa86e738e78684d09dc1d34be08069038a", size = 2837750, upload-time = "2026-08-05T14:15:23.647Z" }, + { url = "https://files.pythonhosted.org/packages/c7/6e/480e2b4c8cfad7271333b44e20e1bcc821632b32b0d5c9c37022ba2e66ee/nvidia_cutlass_dsl_libs_base-4.6.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:669200100131a0b8f2876c535ea532c967335936678593c13aced8273ca7523e", size = 3321233, upload-time = "2026-07-02T03:24:44.902Z" }, + { url = "https://files.pythonhosted.org/packages/3c/88/1f259ffe78178e30a90fbddcec49a567aa077929aec2558f80fcec8dd019/nvidia_cutlass_dsl_libs_base-4.6.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:90a3e7a61d110a8ed005aae83869c6e5dca0723e36298297c0780e21db59c016", size = 2826897, upload-time = "2026-07-02T03:25:06.846Z" }, + { url = "https://files.pythonhosted.org/packages/2c/f8/22653971fcab2a7ed581934f7a2708c9873fa6a8e8eb285422c8eed4ae01/nvidia_cutlass_dsl_libs_base-4.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:6412572899b1c6d182e516b20f2b0a21874ec88d25234e6040fb2a4381de7a1a", size = 3321728, upload-time = "2026-07-02T03:25:28.888Z" }, + { url = "https://files.pythonhosted.org/packages/ce/38/e91f66739d2f8711d1a2457e68cd86d6fbae307ce66ce270a405d4dc6dc7/nvidia_cutlass_dsl_libs_base-4.6.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:e41cd5db4de4b535c30ae9ca4412b957800a62560019ae91fa51cf3ea89bf254", size = 2824817, upload-time = "2026-07-02T03:25:53.814Z" }, + { url = "https://files.pythonhosted.org/packages/e9/5c/0a82b9b2fee054788944d0e7a97b5e63ed0d304969c3b5c4168bed86b71c/nvidia_cutlass_dsl_libs_base-4.6.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:7b5f5502cc827039f42789e1e2ac9aef7010f4d26ef4b9d66f4ab082da0bcd2c", size = 3321628, upload-time = "2026-07-02T03:26:10.487Z" }, + { url = "https://files.pythonhosted.org/packages/7f/67/6c21b2d140bbd1ad94a2e22a0e2881457f9e363bdff1b35898a2d7d25aa2/nvidia_cutlass_dsl_libs_base-4.6.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:7f27357b87c5c797344cca073f1dcf00232aef427daf161adb3cd87b043e37c9", size = 2824911, upload-time = "2026-07-02T03:26:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/9f/7a/afc7477620f7898b6e940f0a6faa58cbcd21e33e6d25031a874bc865f347/nvidia_cutlass_dsl_libs_base-4.6.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a0cde09b71670822baf41e9d02e76883f226ac979b1859ca32fdd51cd34720ea", size = 3321812, upload-time = "2026-07-02T03:26:45.151Z" }, + { url = "https://files.pythonhosted.org/packages/81/f3/72b53467741043e45a2d776485753b37bca6a3466d9964a75aaccccd82db/nvidia_cutlass_dsl_libs_base-4.6.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:777e03b7e1d85085196eaa1512cbc13b7a552ed8b5755e3893e547ca84eaacb7", size = 2824936, upload-time = "2026-07-02T03:27:07.387Z" }, + { url = "https://files.pythonhosted.org/packages/d5/20/c3ed8187e6a69326fc492d98bec31648217f3c6fc74180ae18f9535e8c68/nvidia_cutlass_dsl_libs_base-4.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:87d323cef2c439601f3bcf50a63a9608de322f25adb2ad2a29ea9696657d0e8e", size = 3329994, upload-time = "2026-07-02T03:27:29.421Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4e/69dc9f38b3c9d4ba93e1a71c8b02a29ac1e88f20c57d6b85e2d3e80778bd/nvidia_cutlass_dsl_libs_base-4.6.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:163ed08ea2bbe206e96661ff064e31ece3955e077ed8dd3fb206271333db1610", size = 2837583, upload-time = "2026-07-02T03:27:51.38Z" }, ] [[package]] name = "nvidia-cutlass-dsl-libs-core" -version = "4.6.2" +version = "4.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cuda-python" }, @@ -3961,12 +3999,38 @@ dependencies = [ { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/ef/10c0089234a32f1379e1f213f28f8c7521a9ff0154e3acc58f613d5ef3be/nvidia_cutlass_dsl_libs_core-4.6.2-py3-none-any.whl", hash = "sha256:571d4b46fca1bfbb123d0dc348eaa7fd80af0014ddffc07b849e8195b2364333", size = 772393, upload-time = "2026-08-05T14:09:50.76Z" }, + { url = "https://files.pythonhosted.org/packages/84/94/e4e2404ac06a477096ccf8127bf5d391510d36cafb4be86c8c15b4873b0d/nvidia_cutlass_dsl_libs_core-4.6.0-py3-none-any.whl", hash = "sha256:f9ea6d313a03cb11fa177da32e8747ad0cac51358850810f36aa6c4736192c27", size = 767713, upload-time = "2026-07-02T03:23:39.876Z" }, ] [[package]] name = "nvidia-cutlass-dsl-libs-cu12" -version = "4.6.2" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-python" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "nvidia-cuda-nvdisasm" }, + { name = "nvidia-cutlass-dsl-libs-base" }, + { name = "protobuf" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/c4/ea041a7857b3c7e10e939d787feee59c6c8d2d51a9ae1518684d8894daa1/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:783d5da4aa19a4d11429435419b64264c96d429a1794cbc9df2aa905c1342eee", size = 86992109, upload-time = "2026-07-02T03:28:58.136Z" }, + { url = "https://files.pythonhosted.org/packages/1e/98/4501c0b4053cacbb4e555d306d891f2426ce7edbb148f6e78376418e0356/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:0479904db0736ab912b2f2db7c6a76cf3c9f6e953f94dc5d667f73c27f18d772", size = 88436391, upload-time = "2026-07-02T03:29:21.26Z" }, + { url = "https://files.pythonhosted.org/packages/11/38/62def848b65bf067f434df7680c7e8c48519b25bbd3f03f9cdff3606353b/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:87f132ccc30946949868989f3b1b1adaa714ccdf5c636e5379b54909cc29576c", size = 86992102, upload-time = "2026-07-02T03:29:41.47Z" }, + { url = "https://files.pythonhosted.org/packages/bf/64/f3f8962a9b91dd9368b90e23b2ac81614d6e9df72b55365ec0c216c3f8f9/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:abc341ff0fce40ed0bdadf160f6afac07fb9d01768d4daebd1628c330b3e4210", size = 88436835, upload-time = "2026-07-02T03:30:13.676Z" }, + { url = "https://files.pythonhosted.org/packages/7c/92/773b79f50ca59ca878a5e6be53d7f407deeef56f0b8000bb8cddc2b66d9e/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:45b72e41d343f6b0c1a98669719e03dca4769d7285ae85b7d2a5292168fc73ec", size = 86992268, upload-time = "2026-07-02T03:30:47.112Z" }, + { url = "https://files.pythonhosted.org/packages/07/c1/2521ce3d3f46731d0563bf7e5e0fa6b6ea42c31bcb6763cf4bde16d7b3ce/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:22028842dd9c6064a3de7756b301650be77ddebf6f2acd5336bfbcd05aaf4c02", size = 88437265, upload-time = "2026-07-02T03:31:07.225Z" }, + { url = "https://files.pythonhosted.org/packages/11/35/81556560c4e01c0dbb0746b63529d4513db2e9b0e5f74f641580e3674fda/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:af1428709b6cf37b8aed62f065a708acbd51eb2e68b5828bb8b61cd674ce57ed", size = 86992479, upload-time = "2026-07-02T03:31:33.866Z" }, + { url = "https://files.pythonhosted.org/packages/68/e1/aef0960124c15d7b615e6b0fbd382a6b9650cd4bfb0a1a9eaf2c112fb511/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:18f2aa81d5eeeadcde520b07ce369dc947abf3e29d6655ace9bdedcafb57ac8b", size = 88437698, upload-time = "2026-07-02T03:31:54.343Z" }, + { url = "https://files.pythonhosted.org/packages/68/91/9dc39b2f65715ca47d8c084364650be8fbeeac63affba01d5e1d16ff5a77/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f1dfb31449fe0a24131c53b9d4f957ff6bf3e52ea8709473b0972bf030929e6b", size = 87007515, upload-time = "2026-07-02T03:32:15.241Z" }, + { url = "https://files.pythonhosted.org/packages/17/c3/425b2d64c1da0a1a6017e98d3d2aa69145bf77cf3a095aaa45e065abb4ba/nvidia_cutlass_dsl_libs_cu12-4.6.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:e8dabef7651ce49aa9659f4500a8b43c73325ac08ca1d88d75f383ae8711664d", size = 88455596, upload-time = "2026-07-02T03:32:38.656Z" }, +] + +[[package]] +name = "nvidia-cutlass-dsl-libs-cu13" +version = "4.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cuda-python" }, @@ -3978,16 +4042,16 @@ dependencies = [ { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/42/26/033408d2f1488e941b3434c21e703ec11c42873166af1eb8348271565782/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d245dfb3255d95d1ac75063b758acd3e40d5c6699dc98c80f488165260923e11", size = 87011888, upload-time = "2026-08-05T14:18:09.399Z" }, - { url = "https://files.pythonhosted.org/packages/1a/1b/4bc2842e0b497d9d0fda73976af7640ef742cdb5210e2d143b72c5ec3938/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:4ad5083bf8bcbda83973bdc38a03c2c8452d11fe0afc04b10fc362e23ddd7f8c", size = 88454141, upload-time = "2026-08-05T14:18:31.516Z" }, - { url = "https://files.pythonhosted.org/packages/ad/b7/019a8f74ad30ad9b6190dd5963d8921fc2b03777316e47260eb822624a0a/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ea3b96dd6b627fe20637acfa7876ea1adfa8d81a2b106ffff0bb2d042c0c2731", size = 87013242, upload-time = "2026-08-05T14:18:57.703Z" }, - { url = "https://files.pythonhosted.org/packages/80/6e/70c545a600b52b3f0fbbe01608aefc4e675c924ba886485ad72541f5c88e/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7f510307369d522da8e7d666557b9b9e7df06b94748bd5dc0198d59dfd0d918a", size = 88454261, upload-time = "2026-08-05T14:21:14.733Z" }, - { url = "https://files.pythonhosted.org/packages/e1/54/6006a2b4fcd05f32be451fe8881968dcbed526438b4eae9a527534062c79/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:2f3b163060325777a3847954b3a599556b2bff500eca19806a81d1d635bb4149", size = 87012255, upload-time = "2026-08-05T14:21:44.507Z" }, - { url = "https://files.pythonhosted.org/packages/d1/7a/a08ac0ecdd88bb129a56d95b7c58ea6826e6f5fa00191ef7ecdff8085836/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:84243743382948c83ca627c2dcb106db3844c31d45b8cc2af3d14bc9df0a535e", size = 88453568, upload-time = "2026-08-05T14:22:49.158Z" }, - { url = "https://files.pythonhosted.org/packages/c8/7e/e0f29110f85c129caf2ccf01f0d49e473e6d66c2a220f904870f6093d570/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:37f5d1a7eb8a00a6e303c9ed265ef2586075af49deb9e51865c38275fa1db802", size = 87012154, upload-time = "2026-08-05T14:23:09.963Z" }, - { url = "https://files.pythonhosted.org/packages/16/87/d8af4c8972baaa3b80cbb633fede58f4c8b0d35c8724f1cfa01027103bdb/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0dcb6dd53de145e2b1dde5b2d91a5c30145c97dac9eb18281209703ca9e00a5e", size = 88454125, upload-time = "2026-08-05T14:23:44.62Z" }, - { url = "https://files.pythonhosted.org/packages/26/d6/99ed69f57bde2ee6c1b0c1fd5440944b4ce80476d8799dd7212ea2d175dd/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:38f33d937dd375ee2fef7c802326a18e40bb9083669bb9d1d76462c778e72011", size = 87019480, upload-time = "2026-08-05T14:24:05.561Z" }, - { url = "https://files.pythonhosted.org/packages/46/f8/6b40afd27c89a7cf737c59393a4e9dd49da918ca8eab437f65a19cb3f912/nvidia_cutlass_dsl_libs_cu12-4.6.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:06d8b367aec0ea1a8bca710b92d26df48a2378b16439b46ea8492938454b326a", size = 88464112, upload-time = "2026-08-05T14:24:28.062Z" }, + { url = "https://files.pythonhosted.org/packages/15/7c/2b5c8e98511d9f3f778644a31039d3f69763c05a50574a9d31e8a4a3f2ba/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4876ba55f94f2b1d2285be67ec36e39668cf1ac15b5e7db49830ba5897ea9024", size = 86705152, upload-time = "2026-07-02T03:33:54.958Z" }, + { url = "https://files.pythonhosted.org/packages/68/b8/a44b389a74f67922e11b888b05db7435228a48a69e3d29b3a5ec579ffbdd/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7235c5cfd1db3814bf559d6b976beb759dc7298914f4ff2684a91c3071369598", size = 88026175, upload-time = "2026-07-02T03:34:16.054Z" }, + { url = "https://files.pythonhosted.org/packages/e9/bc/25d542974cc7d2594a22ce1df71c40f8900d44c10aef577cbf5ae7a37e5d/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:8c47899a5778dba4f30de76cce5e22bb5dc2a5bde4979ee1196ec9c6468e80e1", size = 86704408, upload-time = "2026-07-02T03:34:36.929Z" }, + { url = "https://files.pythonhosted.org/packages/b1/0f/bd8b25e6307764a7bfefa519241d6e417f3ba1c75ce548aa76b712a4fd15/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4799fabc4bd1f7825ff00101ae0054c5cfb6769aefd6d9694f6da8c0f07e12c3", size = 88026053, upload-time = "2026-07-02T03:34:59.316Z" }, + { url = "https://files.pythonhosted.org/packages/c9/37/089305ba886ae9ce9359ceba7a8289af744efcaf1b44c85cd3d7c803b9a1/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f4c0835344efb02d4058f812f452fe33dc18c7f19eb94268860162500cdd2354", size = 86704416, upload-time = "2026-07-02T03:35:22.412Z" }, + { url = "https://files.pythonhosted.org/packages/59/1d/e39c5b41e5ca59d5a0809fdfb9ac548430ed957a5e806ad85a77d132c15a/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:c3fd95c840748879ece42b7ecc9585205e4228ebde55c04ecbce48d2987d905a", size = 88026053, upload-time = "2026-07-02T03:35:48.41Z" }, + { url = "https://files.pythonhosted.org/packages/4c/7d/08c00485b5161986834bf27fe446f60157e2025bea3dac453e2cb2c2dac9/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:564a2f643ba4cf9ff93211405f8b75dc3bde20a654c5ac17bf7be309bb059328", size = 86707156, upload-time = "2026-07-02T03:36:27.109Z" }, + { url = "https://files.pythonhosted.org/packages/1e/0e/97edc0011dd226a6ac1359b45469493dcc7c1e6a7627952869f2368da013/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:dfd781fdbed40dd0196ae2ff3747e9039d4f40a4e2e2ee26079c7660ed3ff954", size = 88026622, upload-time = "2026-07-02T03:36:56.109Z" }, + { url = "https://files.pythonhosted.org/packages/4e/5f/16fb431fe0b20d4a1f10ec0ca73a54f235694c7166c7eba278c82017ac9e/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f29150915d972c2310b633659a17906be7aed8f2ad2f90313dbb3041d78aa4fe", size = 86713877, upload-time = "2026-07-02T03:37:29.794Z" }, + { url = "https://files.pythonhosted.org/packages/9b/20/4647d20440aee9f7a0f0c80210bea5b197543d27d1d6a6d1fb70b8273fb1/nvidia_cutlass_dsl_libs_cu13-4.6.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:604ad20b8a741f2df582386e2c912778aefb47fd878a240714a01d4b0e7f0103", size = 88040486, upload-time = "2026-07-02T03:37:58.191Z" }, ] [[package]] @@ -4035,6 +4099,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, ] +[[package]] +name = "nvidia-nvvm" +version = "13.3.73" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/e7/ff646aa6015c7e6d12aad234e68925c87b6681d8d18c3ac40535994a3b0d/nvidia_nvvm-13.3.73-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:0e28e0858a3475e11ac67d35301cd5bf82666a1c0dc4ec4e80ceaf3a5fd1dea8", size = 69250424, upload-time = "2026-06-29T17:08:07.453Z" }, + { url = "https://files.pythonhosted.org/packages/2f/05/35754a7105563fd9b496e5ee8e1acd986aef8258760c3cbccf419aee861a/nvidia_nvvm-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e2bcdd5783b5481445f1f0e7170cb836cc0d72999839ba850bbba6dc97b76bb8", size = 66984478, upload-time = "2026-06-29T17:07:43.765Z" }, +] + +[[package]] +name = "nvtx" +version = "0.2.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/dd/692765e87de30bae1522cdffaa0f2b52949658a92a0fa6d96b1a01eae9d2/nvtx-0.2.15.tar.gz", hash = "sha256:2287d3be05b85661deb386f878d1f536c2e532774aa9ec7a50c434942ed81ae5", size = 121230, upload-time = "2026-03-18T10:01:25.547Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/65/435d10b2041ee082c07d5aed129afd504012c8908796d695f10e66bcc716/nvtx-0.2.15-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:157b80ea9b4db6c8f47f8dbe2fa2e81e7a7f1445bb87f8268f43dec9210b78a1", size = 806443, upload-time = "2026-03-18T10:05:49.308Z" }, + { url = "https://files.pythonhosted.org/packages/47/bc/be94576ba33af75bcc68a857daade64cb86481764d4fb0f36308b1f6fc85/nvtx-0.2.15-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02bca69ee55e0be41eabf908de9dbcdd18e702c7f49f9aa63fd396ce684ff5d5", size = 808183, upload-time = "2026-03-18T10:11:16.262Z" }, + { url = "https://files.pythonhosted.org/packages/c2/07/698355285a03a366ef63ea9762fc1feef3f9f25483e1655408f72d827090/nvtx-0.2.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2cc530cd0f1a2c14a3a7e683833db509888ac5ed4ead94e5c9e2c7317c6937a7", size = 807159, upload-time = "2026-03-18T10:09:49.232Z" }, + { url = "https://files.pythonhosted.org/packages/c0/d1/08f22448d83481408d663065764ba583df091a7de629ed38fc97e522f1af/nvtx-0.2.15-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3ca8030a6d197952318013dd1c12c22da1d4b9feb76ba72e0fcd449961183c2c", size = 806187, upload-time = "2026-03-18T10:13:32.972Z" }, + { url = "https://files.pythonhosted.org/packages/05/c9/8341224b8284f7deb6a634119939de5885adc421e64b6743693b30da2186/nvtx-0.2.15-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d28660d9c46f8ba750d781572b6aa5a1e6221abba224ab32d7fb32c2d0fd67df", size = 780787, upload-time = "2026-03-18T10:10:40.634Z" }, + { url = "https://files.pythonhosted.org/packages/b1/c0/4a5bb7897918de7c7e0191d9342df8ae4cb797ff07276e0f20d13e497ce7/nvtx-0.2.15-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10749686633f880ad53dcdbb2179fad41b45dcf5b7631d4a1070a577577bd386", size = 782575, upload-time = "2026-03-18T10:13:57.3Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/a9acb6d95d2e0e381b2956544768528dd8d7a9e827af8c2014169d838284/nvtx-0.2.15-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:25813ead4fff4d3a6e04f69a72507b096a6bdbecefa369f1100b0e584767bca8", size = 833375, upload-time = "2026-03-18T10:06:31.955Z" }, + { url = "https://files.pythonhosted.org/packages/38/56/c7e8645061cc2fc23f3a54f33e1e340df59216f07dcfb97d46b8ae7dd26c/nvtx-0.2.15-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3741edac4678b92f03d22a3f0a2dfd469f422f85e63db71b038e02525b2404ad", size = 788639, upload-time = "2026-03-18T10:12:01.69Z" }, + { url = "https://files.pythonhosted.org/packages/e0/5b/ca0ba6fa769d08174b7a5b4775c279e2e26611cdd5e7833aa699187871c7/nvtx-0.2.15-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b5171b8283dd3ea9ae688a86d16901b4c2c142c4eb0a4bdbf6c222f5f67f9524", size = 781769, upload-time = "2026-03-18T10:08:59.357Z" }, + { url = "https://files.pythonhosted.org/packages/f7/e1/e02fafc01c18f1868a2d2c030953f49e38d65f2d95884789a6c46ff308f1/nvtx-0.2.15-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3c6d0f27d4f8a2f479eb64a6b842c13aee32120348a1715d995b9bb9f75b35cf", size = 774614, upload-time = "2026-03-18T10:12:46.979Z" }, + { url = "https://files.pythonhosted.org/packages/db/24/528619230976c18364eda2340906ea67b3bf7588b7ce59e054723614abae/nvtx-0.2.15-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aca61135c76b8107ae3c994325613afa661e1336a991c59cc9c6176829b3b32c", size = 834439, upload-time = "2026-03-18T10:05:01.181Z" }, + { url = "https://files.pythonhosted.org/packages/ef/7b/c1b96f13ef89bdf2a8c2f326a97bed89699271990d7c8624fda3fedc6e61/nvtx-0.2.15-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58653bf6fd8453947b9e5153da2ad7aeb0ceafa030de7f133efb3eada5da7ca7", size = 790247, upload-time = "2026-03-18T10:11:39.124Z" }, +] + [[package]] name = "openai" version = "3.0.0" @@ -5061,6 +5154,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/60/5d4751ba3f4a40a6891f24eec885f51afd78d208498268c734e256fb13c4/pydantic_settings-2.12.0-py3-none-any.whl", hash = "sha256:fddb9fd99a5b18da837b29710391e945b1e30c135477f484084ee513adb93809", size = 51880, upload-time = "2025-11-10T14:25:45.546Z" }, ] +[[package]] +name = "pyelftools" +version = "0.33" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/11/767522582afab1b884d277de0e6e011640cb9d7292a38694b4b1a1df1ae8/pyelftools-0.33.tar.gz", hash = "sha256:660d82dcbeb8e83d1702bd97f223f761625da06111c0cc988eac6b8ab0c1b61f", size = 15068655, upload-time = "2026-05-29T12:56:22.553Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/2a/f9697576603dae937727827505a6126a066affb227034e77e6f9068910da/pyelftools-0.33-py3-none-any.whl", hash = "sha256:f215ad5f47d3f1373a21496a6c9e0707c622840d0622f23ff7ce08678b020036", size = 201178, upload-time = "2026-05-29T12:56:20.587Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -5097,6 +5199,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6f/2c/5b079febdc65e1c3fb2729bf958d18b45be7113828528e8a0b5850dd819a/pymdown_extensions-10.21-py3-none-any.whl", hash = "sha256:91b879f9f864d49794c2d9534372b10150e6141096c3908a455e45ca72ad9d3f", size = 268877, upload-time = "2026-02-15T20:44:05.464Z" }, ] +[[package]] +name = "pynvvideocodec" +version = "2.0.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/ce/4559ca81f39b14cc121ed284afc017fe36c3aa40e72ef12170b75d7d2113/pynvvideocodec-2.0.4-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:4dba42331f6d319087d05359787c7c542483fab22107c07b983cad928c1e3cfe", size = 28632422, upload-time = "2026-07-08T04:25:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/2e/86/8766b11b0884fe9c0530870e99c68af1ce68568ebe0652b636caaa9ea50a/pynvvideocodec-2.0.4-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:bd3779ff73ad703393c0a19c3650f269ca25e71902c24efa0719ed4a58cd9390", size = 43180651, upload-time = "2026-05-27T04:02:38.117Z" }, + { url = "https://files.pythonhosted.org/packages/a9/aa/dd826d41581aa6b11a4c922752a89a0dfbfe5f9095de11f8bb10bde46c1d/pynvvideocodec-2.0.4-cp311-cp311-manylinux_2_34_aarch64.whl", hash = "sha256:f809fb18929ac2af042835f10c7679b1c86db9817776f22bc7467907c5c3d918", size = 35755024, upload-time = "2026-05-27T04:03:02.004Z" }, + { url = "https://files.pythonhosted.org/packages/38/1c/78f6fdf85133157a6a3405eab5ef4c2bc8048194dbda1c91bb9b8645bb36/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:bad9e25f494abdcfa8f9dffa33a840509eda3ffcdf6e7cf6465d73be307c0c82", size = 28630316, upload-time = "2026-07-08T04:25:54.596Z" }, + { url = "https://files.pythonhosted.org/packages/ca/49/98da271686e00676f41b1197ba5431ddc341b96d8efb68ea9d68e2b0d870/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b59cec7a1a3f78fad13fead78cad8b6d9686827f9ff4477080245457675a01d0", size = 43176147, upload-time = "2026-05-27T04:04:08.297Z" }, + { url = "https://files.pythonhosted.org/packages/42/80/7b13c12fd5f3243b01190130ce098a44ddf62e030e6ed712911cbfe40311/pynvvideocodec-2.0.4-cp312-cp312-manylinux_2_34_aarch64.whl", hash = "sha256:a0daa28b09705806c8c6b26326df217c45e60c0a12a673ea3ea6ee5e2e7193b0", size = 35754893, upload-time = "2026-05-27T04:04:43.866Z" }, + { url = "https://files.pythonhosted.org/packages/46/d6/8475720d4f0f8fc9e852e0092b308643b71f24d9495ba3bd03c38b16c28d/pynvvideocodec-2.0.4-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:fcb06ef6ef24ee33b8f34950696a4e01636f2d8cdb96c407cd692931d049ac3d", size = 43176124, upload-time = "2026-05-27T04:06:26.415Z" }, + { url = "https://files.pythonhosted.org/packages/3d/28/33d2a4d69b48823801c0b02b1bfbeac04cd9d429ee698d248e7ba946c516/pynvvideocodec-2.0.4-cp313-cp313-manylinux_2_34_aarch64.whl", hash = "sha256:51724c6a0e3623c092cccdf93c8b09cced6881f3d0c76653f6ffbec0371f29cd", size = 35754919, upload-time = "2026-05-27T04:07:09.962Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ce/3062bf51bc0d98a344b9ed229a40535f6f1d309fc9ffaf34719b67ed3e21/pynvvideocodec-2.0.4-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:d9ec06f47bca7b20a6e8234afaf596699c57a477be292613d676802e7e808ed2", size = 43174764, upload-time = "2026-05-27T04:08:19.02Z" }, + { url = "https://files.pythonhosted.org/packages/be/de/27eedeaa3ee5ec3dc2e4b1a7ac2b3d15ebb04cbfa768ca03d159a8328612/pynvvideocodec-2.0.4-cp314-cp314-manylinux_2_34_aarch64.whl", hash = "sha256:834fdbef7f3fc79285b5c2a88d1f1f7cd13543a9a7f2a2786141b181d1daaac9", size = 35755640, upload-time = "2026-05-27T04:08:45.728Z" }, +] + [[package]] name = "pyparsing" version = "3.3.2" @@ -5350,7 +5469,7 @@ wheels = [ [[package]] name = "quack-kernels" -version = "0.6.4" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-tvm-ffi" }, @@ -5359,9 +5478,9 @@ dependencies = [ { name = "torch" }, { name = "torch-c-dlpack-ext" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/5c/67c4a0d54cbb8d4d05af73132d620d1900a193b33e0d5ea6a25829b941d2/quack_kernels-0.6.4.tar.gz", hash = "sha256:8bf08a0e1a85aecc892ce84f4b6ed7abb6a44ec7c68b83040abe174bc8c2b936", size = 845485, upload-time = "2026-08-07T23:25:31.94Z" } +sdist = { url = "https://files.pythonhosted.org/packages/87/6e/589b7e1ac366eaf2f526e350ce9785200f5ef021ad93a92823c5c12cbefc/quack_kernels-0.6.3.tar.gz", hash = "sha256:e307269931e18590f7555afb55debf28e266b4371960aad0528e4055ca8c79a3", size = 839016, upload-time = "2026-08-06T10:28:04.657Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/28/f5/36ec7e075e90bd92bdf30fee0606f189348c2bbb0af6aebaf7382927c622/quack_kernels-0.6.4-py3-none-any.whl", hash = "sha256:e77c5d1f1299b0b38487fe8737df6c6975daa16bca7f7eb883bd1a74d09e7e78", size = 728458, upload-time = "2026-08-07T23:25:30.549Z" }, + { url = "https://files.pythonhosted.org/packages/5c/7c/c16475d19327376f2d5d64ada9f8eff0077fce0b54fd87408b4bb0c04b49/quack_kernels-0.6.3-py3-none-any.whl", hash = "sha256:dfa69468a3f71fb9f7192c087c82e85ad68d59155fa03e408a19c7069fb21b8f", size = 727601, upload-time = "2026-08-06T10:28:03.325Z" }, ] [[package]] @@ -6211,7 +6330,6 @@ dependencies = [ { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "psutil" }, - { name = "setuptools", marker = "python_version < '0'" }, { name = "torch" }, { name = "torch-c-dlpack-ext", marker = "python_full_version < '3.14'" }, { name = "tqdm" }, @@ -6257,6 +6375,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/05/a1/d62dfe7376beaaf1394917e0f8e93ee5f67fea8fcf4107501db35996586b/tokenizers-0.22.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:38337540fbbddff8e999d59970f3c6f35a82de10053206a7562f1ea02d046fa5", size = 10033429, upload-time = "2026-01-05T10:45:14.333Z" }, ] +[[package]] +name = "tokenspeed-mla" +version = "0.1.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "apache-tvm-ffi" }, + { name = "nvidia-cutlass-dsl" }, + { name = "tokenspeed-triton" }, + { name = "torch" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/65/81d7e9f14472bc4c6abb576c9b1edd8e40ab01832027c4e647bfe2890749/tokenspeed_mla-0.1.8-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:952209cf4b29a54e6b6e7e088be9d4a40f24792b06fdfa330de8077b9926a7d9", size = 752341, upload-time = "2026-06-24T03:36:28.619Z" }, + { url = "https://files.pythonhosted.org/packages/27/df/0037ade72b165ac97859040919e006aa3d80cb8cc3a79420fb6c03eb16a0/tokenspeed_mla-0.1.8-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:6a7526d7327746893f8c20d24aa63ba5b8a123d0dfd6e66388e13b768b6452c6", size = 755827, upload-time = "2026-06-24T03:36:29.96Z" }, +] + +[[package]] +name = "tokenspeed-triton" +version = "3.8.10.post20260721" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/ad/748a3b1b3e6559e3008c2dbfa4160c4356e5ff80d427e8eb25881bf42873/tokenspeed_triton-3.8.10.post20260721-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0868b11dc177a44e37cf052f4cf00b55a503672d6bf559689b71f58ebb1fd19d", size = 82963642, upload-time = "2026-07-21T17:14:31.324Z" }, + { url = "https://files.pythonhosted.org/packages/a2/28/f96a19c3c09a9f39a3f86eeb7f64aeed3793bdfa033e406a5f5b1a90b606/tokenspeed_triton-3.8.10.post20260721-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0b66e0587b925e4bfacaf56dd1edbaa149fd6456880387287bb8cda8cdadfbb", size = 87207488, upload-time = "2026-07-21T17:14:34.953Z" }, + { url = "https://files.pythonhosted.org/packages/1d/44/89740db8951918c9acd8731243eef8b44d0eb92ea423552639265c46018e/tokenspeed_triton-3.8.10.post20260721-cp312-abi3-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d793ad0eaebb1d08272c97a2b8f2c31304231748b03de9a08e70a362de92a6e0", size = 82966664, upload-time = "2026-07-21T17:14:38.568Z" }, + { url = "https://files.pythonhosted.org/packages/91/53/f46b401e8ec8998f5b9c39cff0614b796bf49113a09f588cfdfa342789a3/tokenspeed_triton-3.8.10.post20260721-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:66cba8d32a1539afd0ff3eec1782b082d01b4db6824d68017d1d789a03d0be37", size = 87210295, upload-time = "2026-07-21T17:14:42.173Z" }, +] + [[package]] name = "tomli" version = "2.4.0" @@ -6384,6 +6528,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/89/c293d818f9f899db93bf291b42401c05ae29acfb2e53d5341c30ea703e62/torchaudio-2.11.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:67f6edac29ed004652c11db5c19d9debb5d835695930574f564efc8bdd061bba", size = 1771986, upload-time = "2026-03-23T18:13:22.153Z" }, ] +[[package]] +name = "torchcodec" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/2f/1ce2feb161bbd12719f7fd0accef64125f4a6d926e02cd360999b70429e4/torchcodec-0.15.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:7ce1c120275f80eff56842b856ebd48232e46a87c6351607dd8d60ccf197bbb0", size = 2715872, upload-time = "2026-07-15T10:14:02.014Z" }, + { url = "https://files.pythonhosted.org/packages/d8/ee/f14346b9e2d4aac6eee6f1658b5ffa1107e8c1be9e6a2130655a774755c6/torchcodec-0.15.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:1c4257dc64554f1ed848511b7415b772f7df98217a30610d284e2a0a4df1a427", size = 2976495, upload-time = "2026-07-15T10:14:03.439Z" }, + { url = "https://files.pythonhosted.org/packages/9d/de/c00b8d13e3e28de9c76f05b4c25fc4d882b4a3d1451b8d2073d089895684/torchcodec-0.15.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5c62f4257b49c6473b0a1006519274b7daef9ef9c1d66b1a6a025dba9df5daac", size = 2727846, upload-time = "2026-07-15T10:14:08.066Z" }, + { url = "https://files.pythonhosted.org/packages/d0/05/b7ba7ae04db4afeb1fd32d30ec6290d511c374adc464afe191c8fc8d4e22/torchcodec-0.15.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:fa31e33884829332cc55b301aa9d23ba90bf164aa8576a8c68aed6c0061c2d8c", size = 2988620, upload-time = "2026-07-15T10:14:09.504Z" }, + { url = "https://files.pythonhosted.org/packages/af/b9/f9815625b201d41a934f8129d9cf8e956d0b7cccdbc39443538f4f582b38/torchcodec-0.15.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:28c8008494e47c3828eb64b2e9943dbb86d7183c3901a894eda26ce86e01b1a9", size = 2729991, upload-time = "2026-07-15T10:14:13.732Z" }, + { url = "https://files.pythonhosted.org/packages/70/ff/f0e5795100bdf11f6e73a2fcc5197e9010e45030c1ee7f6b3ee32cffefe4/torchcodec-0.15.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:5896a55374d4c90e4788a8eb7c0c7a26a67db5bbe0b6c73975e2bd22f4d98fda", size = 2989745, upload-time = "2026-07-15T10:14:15.194Z" }, + { url = "https://files.pythonhosted.org/packages/b0/07/cbb4a059307fa19fd96aafeb0fd35aeed15bbbff1a7872000734e35c8411/torchcodec-0.15.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:24e8e6a1824cc13986fe678f91b07ac199d19dbc5f2af39decc4966d957b42f5", size = 2732210, upload-time = "2026-07-15T10:14:19.711Z" }, + { url = "https://files.pythonhosted.org/packages/28/57/4bab825dbb8aff755c87ff040fd91e6dbc1bb7c51a9bd5b2fdd61fda0437/torchcodec-0.15.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:b8b0b5b293024e0753455fa141522db794a72ae7b8e1d0a750db5756704ca6ea", size = 2990822, upload-time = "2026-07-15T10:14:21.263Z" }, + { url = "https://files.pythonhosted.org/packages/db/81/cd76bac5183bdcc467f51815d9cf1405bb38a90804b217ce574ac3466162/torchcodec-0.15.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:672aea29b5d9c56dc023e366f40ec4168bfd52f2ac02cf1c07430ad6560fdbe7", size = 2742044, upload-time = "2026-07-15T10:14:25.553Z" }, + { url = "https://files.pythonhosted.org/packages/1c/72/8ab2b9dcd27c1ca7b5c66a6b33fa9acd5cd0cefcc3e778736f8d8a725e9e/torchcodec-0.15.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:7d96bc899819d975db3a38164f4253620fd5683a3b0d447a840cdb6cb091332e", size = 2994598, upload-time = "2026-07-15T10:14:26.991Z" }, +] + [[package]] name = "torchvision" version = "0.26.0" @@ -6661,7 +6822,7 @@ wheels = [ [[package]] name = "vllm" -version = "0.20.0" +version = "0.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -6673,17 +6834,16 @@ dependencies = [ { name = "cloudpickle" }, { name = "compressed-tensors" }, { name = "depyf" }, - { name = "diskcache" }, { name = "einops" }, { name = "fastapi", extra = ["standard"] }, { name = "fastsafetensors" }, { name = "filelock" }, - { name = "flashinfer-cubin" }, { name = "flashinfer-python" }, - { name = "gguf" }, + { name = "humming-kernels", extra = ["cu13"] }, { name = "ijson" }, + { name = "jsonschema" }, { name = "lark" }, - { name = "llguidance", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 'x86_64'" }, + { name = "llguidance", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, { name = "lm-format-enforcer" }, { name = "mcp" }, { name = "mistral-common", extra = ["image"] }, @@ -6694,7 +6854,8 @@ dependencies = [ { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, { name = "nvidia-cudnn-frontend" }, - { name = "nvidia-cutlass-dsl" }, + { name = "nvidia-cutlass-dsl", extra = ["cu13"] }, + { name = "nvtx" }, { name = "openai" }, { name = "openai-harmony" }, { name = "opencv-python-headless" }, @@ -6712,21 +6873,26 @@ dependencies = [ { name = "py-cpuinfo" }, { name = "pybase64" }, { name = "pydantic" }, + { name = "pynvvideocodec" }, { name = "python-json-logger" }, { name = "pyyaml" }, { name = "pyzmq" }, { name = "quack-kernels" }, { name = "regex" }, { name = "requests" }, + { name = "safetensors" }, { name = "sentencepiece" }, { name = "setproctitle" }, { name = "setuptools", marker = "python_full_version >= '3.12'" }, { name = "six", marker = "python_full_version >= '3.12'" }, + { name = "starlette" }, { name = "tiktoken" }, { name = "tilelang" }, { name = "tokenizers" }, + { name = "tokenspeed-mla" }, { name = "torch" }, { name = "torchaudio" }, + { name = "torchcodec" }, { name = "torchvision" }, { name = "tqdm" }, { name = "transformers" }, @@ -6734,10 +6900,10 @@ dependencies = [ { name = "watchfiles" }, { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e7/80/9798ce5e16af5754183ef33a63dc27017e2b51c87f51cc741832ce47a2d5/vllm-0.20.0.tar.gz", hash = "sha256:a6d50152936ee292455af3ffbe359f7a284ac43bf3b68caccf29f368e196cc72", size = 33508260, upload-time = "2026-04-27T11:08:04.666Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/72/fa30f8459d11ae206f1a20bd0ac7ed1b9e390b695fa3dfc9ef6056de0cfe/vllm-0.26.0.tar.gz", hash = "sha256:23e9fa19d7e20ce7dcc1c074d41503e2116d23f19e688f5d5ea91b741f958502", size = 38353572, upload-time = "2026-07-25T10:40:48.095Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/63/5b/26379d3c522379373e50b9f77adf55eb94f4a0f62a6c8e3e7fe3f0bf0d39/vllm-0.20.0-cp38-abi3-manylinux_2_35_aarch64.whl", hash = "sha256:29a135ca0d70650f057f15c7c0b560d24659524c771f70fbddc24597c861c118", size = 235776358, upload-time = "2026-04-27T11:07:22.058Z" }, - { url = "https://files.pythonhosted.org/packages/47/bb/cb02d1e9679fce892a674f86caee25acc9ddd64d7dafa4cfe29e899993a8/vllm-0.20.0-cp38-abi3-manylinux_2_35_x86_64.whl", hash = "sha256:24d28892e210200f6e1bd13f699c42a74cd2bb7364c11248e2348f677c7f6dfb", size = 244415937, upload-time = "2026-04-27T11:07:48.135Z" }, + { url = "https://files.pythonhosted.org/packages/58/27/6ff13689a5931f0c97b7008042f07aacc4a246e7eb06fd9b4d5a72de483c/vllm-0.26.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:52a4c3e55c2c80cc8793e52ccc244457ceade25b0ad7caa1c15e5002a95a1b2c", size = 298269785, upload-time = "2026-07-25T10:40:02.845Z" }, + { url = "https://files.pythonhosted.org/packages/20/96/86edd288415aafc2952bbb969b5ef4e8c58e5525185b60320730276921e6/vllm-0.26.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:adb1e4c9b46d0dfdb094121ae5aad670a42412dd813ed4e5db069ed6a15006de", size = 303698761, upload-time = "2026-07-25T10:40:32.107Z" }, ] [[package]] From 85fa1a5d206dbedcd5b33c6d308ee1b3b96a01be Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Mon, 10 Aug 2026 22:31:45 +0000 Subject: [PATCH 04/28] docs(skill): use TOML inference profiles Signed-off-by: Aaron Gonzales --- skills/anonymizer/SKILL.md | 4 ++-- skills/anonymizer/evals/evals.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/skills/anonymizer/SKILL.md b/skills/anonymizer/SKILL.md index 9fef5d25..c3ade267 100644 --- a/skills/anonymizer/SKILL.md +++ b/skills/anonymizer/SKILL.md @@ -48,7 +48,7 @@ regulatory and business context. - **`risk_tolerance` only applies to Rewrite mode**, not Replace. - **`PrivacyGoal.protect` and `.preserve` must each be 10–1000 chars and at least 3 words.** Be specific (categories, named identifiers, structural facets); avoid generic phrasing like "preserve meaning". - **Validator pool is the only model role with built-in load-spreading.** Set `entity_validator: [a, b, c]` in `models.yaml` if rate limits drop rows. Other roles (rewriter, evaluator, etc.) are single-alias. -- **Self-hosted GLiNER:** When detection must not call `build.nvidia.com` (PHI on-prem, air-gapped, latency), use `tools/inference_service.py` from a **source checkout** to compile a native-GLiNER intent and launch its managed plan. The tool and native server are not installed by `pip install nemo-anonymizer`. Add a provider with `endpoint: http://localhost:8001/v1`, then route `entity_detector` through a `gliner-pii-detector` alias with `provider: local-gliner` and `skip_health_check: true`. Match the compiled host and port in the provider endpoint. `model_configs` is a **complete** model pool, not an overlay. Copy `src/anonymizer/config/default_model_configs/models.yaml` and change only the detector entry, keeping `gpt-oss-120b` and `nemotron-30b-thinking`. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). +- **Self-hosted GLiNER:** When detection must not call `build.nvidia.com` (PHI on-prem, air-gapped, latency), use `tools/inference_service.py` from a **source checkout** to compile the pinned `tools/inference_service_profiles/nvidia-gliner.toml` profile and launch its managed plan. The tool and native server are not installed by `pip install nemo-anonymizer`. Add a provider with `endpoint: http://localhost:8001/v1`, then route `entity_detector` through a `gliner-pii-detector` alias with `provider: local-gliner` and `skip_health_check: true`. Match the compiled host and port in the provider endpoint. `model_configs` is a **complete** model pool, not an overlay. Copy `src/anonymizer/config/default_model_configs/models.yaml` and change only the detector entry, keeping `gpt-oss-120b` and `nemotron-30b-thinking`. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). - **The evaluation judges use their own model roles** (`entity_coverage_judge`, `detection_validity_judge`, `replace_type_fidelity_judge`, `replace_relational_consistency_judge`, `replace_attribute_fidelity_judge`, `rewrite_judge`), configured in the `evaluate` section of `models.yaml`. They are **not** consumed by `preview()` / `run()`, so a config that anonymizes fine can still fail validation at `evaluate()` if those roles are unset. Defaults ship in `src/anonymizer/config/default_model_configs/evaluate.yaml` (`entity_coverage_judge` defaults to `nemotron-super`). - **Verdict columns are null when the judge was unavailable** — `None` means "unscored", never a pass. `entity_coverage` is a `0–1` float (`1.0` = no missed candidate values or no PII found) or `None`; `missed_entities` lists unique candidate values the anonymizer failed to detect. Replace verdict columns (`type_fidelity_valid`, etc.) are `True` / `False` / `None`. Rewrite `detection_valid` is a `0–1` float fraction (or `None` if unscored). Inspect verdicts per record with `evaluated.display_record(i)`. - **`EvaluateConfig` has one knob today: `compute_detection_validity`** (default `False`). Plain `anonymizer.evaluate(result)` runs entity coverage + the mode's quality judges; pass `EvaluateConfig(compute_detection_validity=True)` only to additionally score detection validity (an internal-facing tag-precision metric). @@ -73,7 +73,7 @@ read `docs/troubleshooting.md` or the - **`anonymizer` not installed:** Tell the user `nemo-anonymizer` is not in this Python environment (requires Python ≥ 3.11). Ask if they want you to install it (`pip install nemo-anonymizer`) or do it themselves. Do not install without permission. - **Model/provider setup:** Plain `Anonymizer()` ships with bundled `models.yaml` and `providers.yaml` (see `src/anonymizer/config/default_model_configs/`). For the default path, confirm `NVIDIA_API_KEY` is set. Pass custom `model_configs` or `model_providers` only for non-default endpoints or model pools. See `docs/concepts/models.md` or the [published models guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/models/). - **LLM calls failing at preview:** Check for a missing or invalid API key, a network problem, or a wrong endpoint URL. See `docs/troubleshooting.md` "Validation passed but `preview` errors at LLM call" or the [published troubleshooting guide](https://nvidia-nemo.github.io/Anonymizer/dev/troubleshooting/). -- **Local / on-prem GLiNER:** Clone the Anonymizer repository, compile and launch a native-GLiNER intent with `tools/inference_service.py`, add a provider with the plan's endpoint (normally `http://localhost:8001/v1`), and point `gliner-pii-detector` at `provider: local-gliner` with `skip_health_check: true`. Preflight errors about missing aliases usually mean `model_configs` lists only the detector. Include the full default pool. Use the launch receipt to inspect or cancel the managed process. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). +- **Local / on-prem GLiNER:** Clone the Anonymizer repository, compile and launch `tools/inference_service_profiles/nvidia-gliner.toml` with `tools/inference_service.py`, add a provider with the plan's endpoint (normally `http://localhost:8001/v1`), and point `gliner-pii-detector` at `provider: local-gliner` with `skip_health_check: true`. Preflight errors about missing aliases usually mean `model_configs` lists only the detector. Include the full default pool. Use the launch receipt to inspect or cancel the managed process. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). # Output Template diff --git a/skills/anonymizer/evals/evals.json b/skills/anonymizer/evals/evals.json index 645bc2b8..8c83d096 100644 --- a/skills/anonymizer/evals/evals.json +++ b/skills/anonymizer/evals/evals.json @@ -53,7 +53,7 @@ "expected_skill": "anonymizer", "should_trigger": true, "expected_script": null, - "ground_truth": "The agent loads the anonymizer skill and explains the self-hosted GLiNER path: use tools/inference_service.py from a source checkout to compile a native-GLiNER intent and launch its managed plan, add an OpenAI-compatible provider at the plan endpoint (normally http://localhost:8001/v1), point the gliner-pii-detector/entity_detector alias at provider local-gliner with skip_health_check true, and keep model_configs as a complete model pool copied from defaults rather than a partial overlay.", + "ground_truth": "The agent loads the anonymizer skill and explains the self-hosted GLiNER path: use tools/inference_service.py from a source checkout to compile the pinned tools/inference_service_profiles/nvidia-gliner.toml profile and launch its managed plan, add an OpenAI-compatible provider at the plan endpoint (normally http://localhost:8001/v1), point the gliner-pii-detector/entity_detector alias at provider local-gliner with skip_health_check true, and keep model_configs as a complete model pool copied from defaults rather than a partial overlay.", "expected_behavior": [ "The agent read skills/anonymizer/SKILL.md before answering", "The answer says the reference GLiNER server comes from a source checkout, not pip-installed package files", From a0275ad5511a732f770a5957807e055c8e4ab323 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 11 Aug 2026 17:01:40 +0000 Subject: [PATCH 05/28] fix(dev): preserve FastAPI request runtime type Signed-off-by: Aaron Gonzales --- tests/tools/test_native_gliner.py | 8 ++++++++ .../inference_service_compiler/native_gliner.py | 17 +++++++++++------ 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/tests/tools/test_native_gliner.py b/tests/tools/test_native_gliner.py index 9e1f1dc5..f8ea48c1 100644 --- a/tests/tools/test_native_gliner.py +++ b/tests/tools/test_native_gliner.py @@ -12,6 +12,7 @@ import sys from pathlib import Path from types import ModuleType +from typing import get_type_hints import pytest @@ -220,6 +221,13 @@ def test_fastapi_app_alias_is_preserved(monkeypatch: pytest.MonkeyPatch) -> None assert server.app is server.api +def test_fastapi_route_uses_concrete_request_type(monkeypatch: pytest.MonkeyPatch) -> None: + """FastAPI resolves the endpoint parameter to its runtime Request class.""" + server = load_server(monkeypatch) + + assert get_type_hints(server.chat_completions)["request"] is object + + def test_chat_completion_uses_anonymizer_json_string_contract(monkeypatch: pytest.MonkeyPatch) -> None: """The OpenAI response embeds flat entities in `message.content` JSON.""" server = load_server(monkeypatch) diff --git a/tools/inference_service_compiler/native_gliner.py b/tools/inference_service_compiler/native_gliner.py index 142a7cb9..e3c58db2 100755 --- a/tools/inference_service_compiler/native_gliner.py +++ b/tools/inference_service_compiler/native_gliner.py @@ -34,7 +34,7 @@ from contextlib import asynccontextmanager from dataclasses import dataclass from enum import StrEnum -from typing import Protocol, cast +from typing import TYPE_CHECKING, Protocol, cast import structlog # ty: ignore[unresolved-import] -- optional PEP 723 server dependency import uvicorn @@ -134,11 +134,16 @@ def infer(self, chunks: list[str], params: DetectParams) -> list[list[Entity]]: """Detect normalized entities for each chunk.""" -class RequestLike(Protocol): - """Request surface consumed by the OpenAI-compatible endpoint.""" +if TYPE_CHECKING: - async def json(self) -> object: - """Decode the request body.""" + class FastAPIRequest(Protocol): + """Request surface consumed by the OpenAI-compatible endpoint.""" + + async def json(self) -> object: + """Decode the request body.""" + +else: + FastAPIRequest = _fastapi.Request class NvidiaModel(Protocol): @@ -617,7 +622,7 @@ def list_models() -> dict[str, object]: @api.post("/v1/chat/completions") -async def chat_completions(request: RequestLike) -> dict[str, object]: +async def chat_completions(request: FastAPIRequest) -> dict[str, object]: """Detect requested entity labels and return Anonymizer's JSON-string content.""" if detector is None: raise _fastapi.HTTPException(status_code=503, detail="GLiNER model is not loaded") From a5afac94768dba2c7445a56d379edbbea70ea516 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 11 Aug 2026 21:17:20 +0000 Subject: [PATCH 06/28] feat(dev): integrate external vLLM Factory Signed-off-by: Aaron Gonzales --- README.md | 3 +- docs/concepts/inference-services.md | 73 +++- docs/concepts/models.md | 5 +- docs/concepts/self-hosting-gliner.md | 106 ++++-- pyproject.toml | 1 + skills/anonymizer/SKILL.md | 4 +- skills/anonymizer/evals/evals.json | 6 +- tests/tools/test_inference_service.py | 60 +++- tests/tools/test_vllm_factory.py | 73 +++- tests/tools/test_vllm_factory_adapter.py | 132 +++++++ tests/tools/test_vllm_factory_integration.py | 64 ++++ tools/inference_service_compiler/compiler.py | 86 ++++- tools/inference_service_compiler/models.py | 8 + .../vllm_factory_adapter.py | 334 ++++++++++++++++++ .../vllm_factory_integration.py | 132 +++++++ .../{vllm_factory.py => vllm_runtime.py} | 59 +++- .../inference_service_compiler/vllm_server.py | 4 +- tools/inference_service_profiles/gliner2.toml | 11 +- .../nvidia-gliner.toml | 11 +- uv.lock | 82 ++++- 20 files changed, 1160 insertions(+), 94 deletions(-) create mode 100644 tests/tools/test_vllm_factory_adapter.py create mode 100644 tests/tools/test_vllm_factory_integration.py create mode 100644 tools/inference_service_compiler/vllm_factory_adapter.py create mode 100644 tools/inference_service_compiler/vllm_factory_integration.py rename tools/inference_service_compiler/{vllm_factory.py => vllm_runtime.py} (67%) diff --git a/README.md b/README.md index 6088d38a..b18253ce 100644 --- a/README.md +++ b/README.md @@ -157,7 +157,8 @@ make install-pre-commit # Install pre-commit hooks ### Local inference services Use the source-tree inference service compiler to create immutable plans and -managed launch receipts for native GLiNER or vLLM processes and containers: +managed launch receipts for GLiNER or GLiNER2 through the pinned external vLLM +Factory project, the native detector fallback, and vLLM generation: ```bash uv run tools/inference_service.py compile --profile tools/inference_service_profiles/nvidia-gliner.toml --source-revision 3f68c145 --output plan.json diff --git a/docs/concepts/inference-services.md b/docs/concepts/inference-services.md index 4805f043..601e2336 100644 --- a/docs/concepts/inference-services.md +++ b/docs/concepts/inference-services.md @@ -12,7 +12,10 @@ and known effects of the operation. The tool is source-owned under `tools/` and is not included in the `nemo-anonymizer` wheel. It currently supports: -- entity detection with the native NVIDIA GLiNER or GLiNER2 runtime; +- entity detection with NVIDIA GLiNER or GLiNER2 through the external + [vLLM Factory](https://github.com/latenceainew/vllm-factory) project; +- entity detection with the source-owned native runtime when a GPU vLLM stack + is not appropriate; - generation through a vLLM-compatible model; - managed local processes and managed local Docker containers; and - direct HTTP access to the resulting OpenAI-compatible endpoint. @@ -37,11 +40,26 @@ binds each plan to its exact intent, command, endpoint contract, compatibility evidence, and source revision. Runtime commands reject a changed plan before performing effects. -## Native GLiNER +## GLiNER through vLLM Factory -The source tree includes pinned profiles for NVIDIA GLiNER and GLiNER2 under -`tools/inference_service_profiles/`. Compile and launch one from the repository -root. Replace the example source revision with the revision of your checkout: +The pinned NVIDIA GLiNER and GLiNER2 profiles under +`tools/inference_service_profiles/` use vLLM Factory. Install the local model +group on a Linux GPU host: + +```bash +uv sync --group dev --group local-models +python -m vllm_factory.compat.doctor +nvidia-smi +``` + +The dependency group pins vLLM 0.26.0 and an exact vLLM Factory source commit. +The compiled plan records both dependencies. The runtime calls vLLM Factory's +model-preparation Python API with the profile's pinned Hugging Face revision, +loads its GLiNER model plugin and IOProcessor, then constructs the vLLM server +through vLLM's Python API. It does not invoke either project's CLI. + +Compile and launch the NVIDIA profile from the repository root. Replace the +example source revision with the revision of your checkout: ```bash uv run tools/inference_service.py compile \ @@ -54,14 +72,41 @@ uv run tools/inference_service.py launch \ --output gliner-launch.json ``` -The launch returns only after `/v1/models` and an entity-detection contract -probe succeed. The receipt records the process ID plus its Linux start marker +The service keeps vLLM Factory's native `POST /pooling` endpoint. A thin +in-process adapter also exposes Anonymizer's `POST /v1/chat/completions` +detector contract. The adapter preserves dynamic labels, character offsets, +scores, overlapping character chunks, and label-free DataDesigner health +checks. Model preparation, scheduling, batching, inference, and decoding stay +inside vLLM Factory and vLLM. + +Launch returns only after `/v1/models` and a positive entity-detection contract +probe succeed. The receipt records the process ID plus its Linux start marker, when available, which lets a later invocation guard against PID reuse. Use `tools/inference_service_profiles/gliner2.toml` for the pinned GLiNER2 -checkpoint. Entity detection is intentionally native. The compiler rejects a -GLiNER profile paired with vLLM because vLLM 0.26 does not expose GLiNER's -dynamic-label, offset, and score contract. +checkpoint and the `deberta_gliner2` plugin. The NVIDIA profile uses +`deberta_gliner`. Stock vLLM remains invalid for entity detection unless the +intent selects one of these characterized factory integrations. + +vLLM Factory detection is characterized as a managed local process. The +compiler rejects its use with the stock vLLM Docker image because that image +does not contain the pinned external project or Anonymizer's protocol adapter. + +## Native GLiNER fallback + +The source-owned native runtime remains available for CPU, MPS, and local GPU +use. Select `kind = "native-gliner"` and choose the family in a custom profile: + +```toml +[engine] +kind = "native-gliner" +family = "nvidia-gliner" # or "gliner2" +device = "auto" +``` + +This path runs `tools/inference_service_compiler/native_gliner.py` as an +isolated uv script. It preserves the same OpenAI-compatible detector contract +but does not use vLLM Factory's scheduler or IOProcessor plugins. ## Local vLLM Process @@ -72,7 +117,7 @@ uv sync --group dev --group local-models nvidia-smi ``` -The `local-models` group pins vLLM 0.26.0. The local plan starts +The `local-models` group pins vLLM 0.26.0. The local generation plan starts `tools/inference_service_compiler/vllm_server.py`, which constructs vLLM's frontend and async engine through its Python API. It does not invoke `vllm serve` or inherit vLLM's full CLI surface. @@ -108,7 +153,7 @@ name = "privacy" The compiler renders the corresponding vLLM `--lora-modules` arguments. -## Docker vLLM +## Docker vLLM generation Change the placement to Docker to use vLLM's official OpenAI-compatible image: @@ -168,8 +213,8 @@ Use `launch --log-directory PATH` to select another location. ## Connect Anonymizer -Both native GLiNER and vLLM expose OpenAI-compatible URLs. Add the compiled -endpoint to a custom provider file: +The factory-backed detector, native detector, and generation server expose +OpenAI-compatible URLs. Add the compiled endpoint to a custom provider file: ```yaml title="providers.yaml" providers: diff --git a/docs/concepts/models.md b/docs/concepts/models.md index 700294f1..690cb1d9 100644 --- a/docs/concepts/models.md +++ b/docs/concepts/models.md @@ -36,7 +36,10 @@ Each pipeline stage has a **role** mapped to one of these aliases. See the full Pass `model_providers` when you need a non-default endpoint — for example OpenAI, OpenRouter, a local GLiNER server, or an internal inference deployment. Plain `Anonymizer()` already uses bundled [build.nvidia.com](https://build.nvidia.com) settings; override only when your models point at a different provider name or URL. -For managed native GLiNER and vLLM endpoints, see [Run local inference services](inference-services.md). That guide covers immutable plans, local processes, Docker, capability receipts, and provider configuration. +For managed GLiNER through vLLM Factory, the native detector fallback, and +vLLM generation endpoints, see [Run local inference services](inference-services.md). +That guide covers immutable plans, local processes, Docker, capability +receipts, and provider configuration. Set your API keys first: diff --git a/docs/concepts/self-hosting-gliner.md b/docs/concepts/self-hosting-gliner.md index 9aaef969..2dd60d2b 100644 --- a/docs/concepts/self-hosting-gliner.md +++ b/docs/concepts/self-hosting-gliner.md @@ -5,9 +5,16 @@ By default, Anonymizer's entity detection stage calls the hosted `nvidia/gliner-pii` model on `build.nvidia.com`. For PHI-sensitive workloads that cannot leave the host, or latency-critical setups, you can serve GLiNER locally instead. -The default NVIDIA GLiNER model is small (~500 MB) and runs comfortably on CPU — making it a good fit to run alongside a local LLM without competing for GPU memory. It also runs on GPU if one is available, which cuts detection latency on long documents. The optional GLiNER2 PII model is also fully local and supports GPU or CPU inference. +The default NVIDIA GLiNER model is small enough to share a GPU with a local +LLM. The optional GLiNER2 PII model is also fully local. The pinned reference +profiles serve both models through vLLM 0.26 and the external +[vLLM Factory](https://github.com/latenceainew/vllm-factory) project. A native +CPU, MPS, or GPU fallback remains available for custom profiles. -The characterized native server lives inside the source-tree [inference service compiler](inference-services.md). It is **not** installed with `pip install nemo-anonymizer`; compile and launch it from a source checkout. +The characterized services live inside the source-tree +[inference service compiler](inference-services.md). They are **not** installed +with `pip install nemo-anonymizer`; compile and launch them from a source +checkout. --- @@ -52,31 +59,38 @@ Long inputs are split into overlapping chunks before inference. A self-hosted se ## Reference implementation -The native GLiNER runtime compiled by `tools/inference_service.py` implements the contract above. Its internal server module defaults to `nvidia/gliner-pii`, also supports the local PII-capable `fastino/gliner2-privacy-filter-PII-multi` model, exposes `POST /v1/chat/completions` (and `GET /v1/models`), and uses two levels of batching: +The pinned profiles compile a vLLM Factory integration. The external project +prepares each GLiNER checkpoint for vLLM, registers the model implementation, +preprocesses requests through an IOProcessor, schedules pooling inference, and +decodes the output. Its native endpoint is `POST /pooling`. -1. **Chunk batching** — long text is split into overlapping windows; all chunks are passed to one runtime batch call. -2. **Request coalescing** (optional, on by default) — concurrent HTTP requests from DataDesigner are grouped briefly, then all their chunks are inferred together. +Anonymizer adds a thin middleware function inside the same vLLM process. It +translates `POST /v1/chat/completions` into in-process pooling calls and restores +the detector response shape. The adapter does not load model weights or run +inference itself. It handles two wire-level responsibilities: -```python title="tools/inference_service_compiler/native_gliner.py (excerpt)" -@api.post("/v1/chat/completions") -async def chat_completions(request: Request): - body = require_mapping(await request.json(), "request") - params = parse_detect_params(body) - text = extract_text(body.get("messages", [])) - entities = await detector.detect(text, params) - ... -``` +1. **Chunk submission**: long text is split into overlapping character windows, + then each window enters vLLM's scheduler. +2. **Response normalization**: GLiNER and GLiNER2 results become one list of + entities with document offsets and overlap deduplication. -When `flat_ner` is `false` (Anonymizer's default), the server removes nested subset spans before score-based deduplication across chunk overlaps. +```python title="tools/inference_service_compiler/vllm_factory_adapter.py (excerpt)" +results = await asyncio.gather( + *(invoke_pooling(handler=handler, text=chunk, ...) for chunk, offset in chunks) +) +entities = merge_entities(plugin=plugin, chunks=chunks, results=results, ...) +``` -| Environment variable | Default | Purpose | -|---|---|---| -| `DEVICE` | `auto` | `auto`, `cuda`, `cpu`, or `mps` (Apple Silicon GPU) | -| `GLINER_BATCH_MODE` | `true` | Coalesce concurrent HTTP requests before inference | -| `GLINER_MAX_BATCH_REQUESTS` | `32` | Max requests per coalesced batch | -| `GLINER_BATCH_WAIT_MS` | `10` | Max wait time to fill a batch (milliseconds) | +When `flat_ner` is `false` (Anonymizer's default), the adapter removes nested +subset spans before score-based deduplication across chunk overlaps. A request +without `labels` returns an empty entity list so DataDesigner's generic model +health check can validate the endpoint without running meaningless inference. -Set `GLINER_BATCH_MODE=false` to disable request coalescing; chunk batching still runs per request. +The native fallback still lives at +`tools/inference_service_compiler/native_gliner.py`. It has its own uv-managed +dependencies and supports `DEVICE`, `GLINER_BATCH_MODE`, +`GLINER_MAX_BATCH_REQUESTS`, and `GLINER_BATCH_WAIT_MS`. See +[Native GLiNER fallback](inference-services.md#native-gliner-fallback). --- @@ -88,18 +102,32 @@ Set `GLINER_BATCH_MODE=false` to disable request coalescing; chunk batching stil ### Dependencies -The managed native server is a [PEP 723](https://peps.python.org/pep-0723/) uv script and declares its own Python 3.13+ dependencies. Install [uv](https://docs.astral.sh/uv/); launch resolves the isolated environment for the selected local runtime. The first environment setup can be large because the runtime packages include Torch and its platform dependencies. No package installation in the Anonymizer environment is required. +Install [uv](https://docs.astral.sh/uv/) and sync the local model group on a +Linux GPU host: + +```bash +uv sync --group dev --group local-models +python -m vllm_factory.compat.doctor +``` + +The group pins `vllm==0.26.0` and vLLM Factory at the exact source revision +recorded in `pyproject.toml` and `uv.lock`. The doctor must report the general +plugins group, IOProcessor plugins group, and native IO mode. -On first launch, the selected public checkpoint is downloaded from Hugging Face and cached under `~/.cache/huggingface/`. No Hugging Face token is required. Package and checkpoint setup use the network; inference stays local and the server does not call a remote inference service after setup. +On first launch, the selected public checkpoint is downloaded from Hugging Face +and cached under `~/.cache/huggingface/`. The integration supplies the TOML +profile's immutable revision to vLLM Factory's preparation API and writes a +provenance record beside the prepared model. Package and checkpoint setup use +the network; inference stays local after setup. ### Start the server -Compile the pinned native GLiNER TOML profile and launch the resulting plan as -shown in [Run local inference services](inference-services.md#native-gliner). -The profile keeps the model checkpoint, engine family, device, placement, -access, and managed lifecycle separate. `nvidia-gliner` is the default engine -family and `nvidia/gliner-pii` is the default model; GLiNER2 uses the -`fastino/gliner2-privacy-filter-PII-multi` checkpoint. +Compile the pinned vLLM Factory GLiNER TOML profile and launch the resulting +plan as shown in +[Run local inference services](inference-services.md#gliner-through-vllm-factory). +The profile keeps the model checkpoint, engine, factory plugin, placement, +access, and managed lifecycle separate. NVIDIA GLiNER uses +`deberta_gliner`; GLiNER2 uses `deberta_gliner2`. Launch writes a versioned receipt only after the model-list and detection contract probes pass. Use that receipt with the compiler's `inspect` and @@ -107,7 +135,10 @@ contract probes pass. Use that receipt with the compiler's `inspect` and The model families do not use identical label vocabularies. The request example below targets the default NVIDIA model and uses `user_name`; the default GLiNER2 PII checkpoint uses `username` for that category. -The reference server has **no authentication**. The default bind address is `127.0.0.1` so detection traffic stays on localhost. Use `--host 0.0.0.0` only when Anonymizer runs on another host in a trusted environment, ideally behind authentication and TLS termination. +The reference profiles have **no authentication**. The default bind address is +`127.0.0.1` so detection traffic stays on localhost. Set `api_key_env` in the +engine or place the endpoint behind authenticated TLS before exposing it to +another host. Verify the server is reachable: @@ -177,7 +208,7 @@ model_configs: model: nvidia/gliner-pii provider: local-gliner inference_parameters: - max_parallel_requests: 8 # send concurrent rows; the reference server batches them + max_parallel_requests: 8 # vLLM continuously batches concurrent detector calls timeout: 120 - alias: gpt-oss-120b @@ -214,9 +245,12 @@ anonymizer = Anonymizer( ## Performance notes -- **Batch mode**: The reference server coalesces concurrent detector requests by default. Pair it with a higher `max_parallel_requests` on the `gliner-pii-detector` alias (see YAML above) so DataDesigner sends multiple rows at once and the server fills GPU batches efficiently. -- On CPU, detection of a ~1000-character note with ~30 candidate labels takes **5–20 ms** per request on a modern x86 core. For typical Anonymizer workflows this is a rounding error compared to the LLM roles that follow, and keeping GLiNER on CPU frees GPU memory for the LLM. -- On GPU the same request drops to roughly **1–3 ms** — worth it when you're processing tens of thousands of documents in a batch workflow, or when the host has spare GPU memory next to the LLM. -- Choose device with the `DEVICE` environment variable (`auto`, `cuda`, `mps`, `cpu`). `auto` prefers Apple Silicon GPU (MPS), then NVIDIA CUDA, then CPU. +- **Continuous batching**: Pair vLLM Factory with a higher + `max_parallel_requests` on the detector alias so DataDesigner supplies enough + concurrent work for vLLM's scheduler. +- **GPU memory**: `gpu_memory_utilization` is a vLLM memory budget. Size it with + any colocated generation service in mind. +- **Native fallback**: Use the native engine when CPU or MPS deployment matters + more than vLLM scheduling. - The default GLiNER threshold is `0.3`. Lower values detect more spans (higher recall, more false positives); higher values improve precision but miss edge cases. Tune via `Detect(gliner_threshold=...)`. - Each request loads the FULL list of candidate labels passed from `Detect.entity_labels`. If you only need a subset (e.g. a clinical-only deployment), narrowing that list materially speeds up detection. diff --git a/pyproject.toml b/pyproject.toml index 32a66c85..9af1203e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ notebooks = [ ] local-models = [ "vllm==0.26.0; sys_platform == 'linux'", + "vllm-factory[gliner] @ git+https://github.com/latenceainew/vllm-factory.git@7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0; sys_platform == 'linux'", ] [build-system] diff --git a/skills/anonymizer/SKILL.md b/skills/anonymizer/SKILL.md index c3ade267..b072b523 100644 --- a/skills/anonymizer/SKILL.md +++ b/skills/anonymizer/SKILL.md @@ -48,7 +48,7 @@ regulatory and business context. - **`risk_tolerance` only applies to Rewrite mode**, not Replace. - **`PrivacyGoal.protect` and `.preserve` must each be 10–1000 chars and at least 3 words.** Be specific (categories, named identifiers, structural facets); avoid generic phrasing like "preserve meaning". - **Validator pool is the only model role with built-in load-spreading.** Set `entity_validator: [a, b, c]` in `models.yaml` if rate limits drop rows. Other roles (rewriter, evaluator, etc.) are single-alias. -- **Self-hosted GLiNER:** When detection must not call `build.nvidia.com` (PHI on-prem, air-gapped, latency), use `tools/inference_service.py` from a **source checkout** to compile the pinned `tools/inference_service_profiles/nvidia-gliner.toml` profile and launch its managed plan. The tool and native server are not installed by `pip install nemo-anonymizer`. Add a provider with `endpoint: http://localhost:8001/v1`, then route `entity_detector` through a `gliner-pii-detector` alias with `provider: local-gliner` and `skip_health_check: true`. Match the compiled host and port in the provider endpoint. `model_configs` is a **complete** model pool, not an overlay. Copy `src/anonymizer/config/default_model_configs/models.yaml` and change only the detector entry, keeping `gpt-oss-120b` and `nemotron-30b-thinking`. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). +- **Self-hosted GLiNER:** When detection must not call `build.nvidia.com` (PHI on-prem, air-gapped, latency), use `tools/inference_service.py` from a **source checkout** to compile the pinned `tools/inference_service_profiles/nvidia-gliner.toml` profile and launch its managed plan. The profile uses vLLM 0.26 and the external vLLM Factory project at a pinned source revision. The tool and runtime are not installed by `pip install nemo-anonymizer`; install the repository's `local-models` dependency group first. Add a provider with `endpoint: http://localhost:8001/v1`, then route `entity_detector` through a `gliner-pii-detector` alias with `provider: local-gliner`. Match the compiled host and port in the provider endpoint. `model_configs` is a **complete** model pool, not an overlay. Copy `src/anonymizer/config/default_model_configs/models.yaml` and change only the detector entry, keeping `gpt-oss-120b` and `nemotron-30b-thinking`. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). - **The evaluation judges use their own model roles** (`entity_coverage_judge`, `detection_validity_judge`, `replace_type_fidelity_judge`, `replace_relational_consistency_judge`, `replace_attribute_fidelity_judge`, `rewrite_judge`), configured in the `evaluate` section of `models.yaml`. They are **not** consumed by `preview()` / `run()`, so a config that anonymizes fine can still fail validation at `evaluate()` if those roles are unset. Defaults ship in `src/anonymizer/config/default_model_configs/evaluate.yaml` (`entity_coverage_judge` defaults to `nemotron-super`). - **Verdict columns are null when the judge was unavailable** — `None` means "unscored", never a pass. `entity_coverage` is a `0–1` float (`1.0` = no missed candidate values or no PII found) or `None`; `missed_entities` lists unique candidate values the anonymizer failed to detect. Replace verdict columns (`type_fidelity_valid`, etc.) are `True` / `False` / `None`. Rewrite `detection_valid` is a `0–1` float fraction (or `None` if unscored). Inspect verdicts per record with `evaluated.display_record(i)`. - **`EvaluateConfig` has one knob today: `compute_detection_validity`** (default `False`). Plain `anonymizer.evaluate(result)` runs entity coverage + the mode's quality judges; pass `EvaluateConfig(compute_detection_validity=True)` only to additionally score detection validity (an internal-facing tag-precision metric). @@ -73,7 +73,7 @@ read `docs/troubleshooting.md` or the - **`anonymizer` not installed:** Tell the user `nemo-anonymizer` is not in this Python environment (requires Python ≥ 3.11). Ask if they want you to install it (`pip install nemo-anonymizer`) or do it themselves. Do not install without permission. - **Model/provider setup:** Plain `Anonymizer()` ships with bundled `models.yaml` and `providers.yaml` (see `src/anonymizer/config/default_model_configs/`). For the default path, confirm `NVIDIA_API_KEY` is set. Pass custom `model_configs` or `model_providers` only for non-default endpoints or model pools. See `docs/concepts/models.md` or the [published models guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/models/). - **LLM calls failing at preview:** Check for a missing or invalid API key, a network problem, or a wrong endpoint URL. See `docs/troubleshooting.md` "Validation passed but `preview` errors at LLM call" or the [published troubleshooting guide](https://nvidia-nemo.github.io/Anonymizer/dev/troubleshooting/). -- **Local / on-prem GLiNER:** Clone the Anonymizer repository, compile and launch `tools/inference_service_profiles/nvidia-gliner.toml` with `tools/inference_service.py`, add a provider with the plan's endpoint (normally `http://localhost:8001/v1`), and point `gliner-pii-detector` at `provider: local-gliner` with `skip_health_check: true`. Preflight errors about missing aliases usually mean `model_configs` lists only the detector. Include the full default pool. Use the launch receipt to inspect or cancel the managed process. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). +- **Local / on-prem GLiNER:** Clone the Anonymizer repository, install its `local-models` group, then compile and launch `tools/inference_service_profiles/nvidia-gliner.toml` with `tools/inference_service.py`. Add a provider with the plan's endpoint (normally `http://localhost:8001/v1`) and point `gliner-pii-detector` at `provider: local-gliner`. The vLLM Factory adapter supports DataDesigner's health check, so do not suppress it. Preflight errors about missing aliases usually mean `model_configs` lists only the detector. Include the full default pool. Use the launch receipt to inspect or cancel the managed process. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). # Output Template diff --git a/skills/anonymizer/evals/evals.json b/skills/anonymizer/evals/evals.json index 8c83d096..840ad034 100644 --- a/skills/anonymizer/evals/evals.json +++ b/skills/anonymizer/evals/evals.json @@ -53,13 +53,13 @@ "expected_skill": "anonymizer", "should_trigger": true, "expected_script": null, - "ground_truth": "The agent loads the anonymizer skill and explains the self-hosted GLiNER path: use tools/inference_service.py from a source checkout to compile the pinned tools/inference_service_profiles/nvidia-gliner.toml profile and launch its managed plan, add an OpenAI-compatible provider at the plan endpoint (normally http://localhost:8001/v1), point the gliner-pii-detector/entity_detector alias at provider local-gliner with skip_health_check true, and keep model_configs as a complete model pool copied from defaults rather than a partial overlay.", + "ground_truth": "The agent loads the anonymizer skill and explains the self-hosted GLiNER path: use tools/inference_service.py from a source checkout with the local-models dependency group to compile the pinned tools/inference_service_profiles/nvidia-gliner.toml vLLM Factory profile and launch its managed plan, add an OpenAI-compatible provider at the plan endpoint (normally http://localhost:8001/v1), point the gliner-pii-detector/entity_detector alias at provider local-gliner, retain the health check supported by the adapter, and keep model_configs as a complete model pool copied from defaults rather than a partial overlay.", "expected_behavior": [ "The agent read skills/anonymizer/SKILL.md before answering", - "The answer says the reference GLiNER server comes from a source checkout, not pip-installed package files", + "The answer says the compiler and adapter come from a source checkout and require the local-models dependency group", "The answer includes the localhost OpenAI-compatible endpoint", "The answer routes entity_detector through the gliner-pii-detector alias using a local provider", - "The answer sets or mentions skip_health_check for the local GLiNER provider", + "The answer identifies the pinned external vLLM Factory integration and keeps the supported health check enabled", "The answer warns that model_configs must include the full default model pool", "The answer cites the self-hosting GLiNER docs with the published Anonymizer docs fallback" ] diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index fc171ce9..4a258d38 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -197,7 +197,7 @@ def test_compile_vllm_docker_plan_keeps_secrets_symbolic() -> None: def test_compile_local_vllm_plan_uses_the_python_server_factory() -> None: - """Local vLLM runs through the source-owned Python factory, not its CLI binary.""" + """Local vLLM runs through the source-owned Python runtime, not its CLI binary.""" models, compiler = load_compiler_modules() intent = models.InferenceIntent( task=models.Generation(chat=True), @@ -219,8 +219,32 @@ def test_compile_local_vllm_plan_uses_the_python_server_factory() -> None: assert VLLM_SERVER_PATH.is_file() -def test_compiler_rejects_gliner_through_vllm() -> None: - """GLiNER models stay on their characterized native runtime.""" +def test_compiler_accepts_gliner_through_external_vllm_factory() -> None: + """GLiNER compiles through the pinned factory plugin and Python vLLM runtime.""" + models, compiler = load_compiler_modules() + intent = models.InferenceIntent( + task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), + model=models.HuggingFaceModel(model_id="nvidia/gliner-pii", revision="bd23e8ef4425fd04"), + engine=models.VllmEngine(factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), + placement=models.LocalProcessPlacement(), + access=models.DirectAccess(), + lifecycle=models.ManagedLifecycle(), + ) + + plan = compiler.compile_intent(intent, source_revision="3f68c145") + + assert plan.declared_capabilities == ("dynamic-labels", "offsets", "scores") + assert "vllm==0.26.0" in plan.dependencies + assert ( + "vllm-factory[gliner] @ git+https://github.com/latenceainew/vllm-factory.git@" + "7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0" + ) in plan.dependencies + assert "--vllm-factory-plugin" in plan.command.render_argv() + assert "deberta_gliner" in plan.command.render_argv() + + +def test_compiler_rejects_gliner_through_stock_vllm() -> None: + """Entity detection requires an explicit vLLM Factory plugin.""" models, compiler = load_compiler_modules() intent = models.InferenceIntent( task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), @@ -235,10 +259,32 @@ def test_compiler_rejects_gliner_through_vllm() -> None: compiler.compile_intent(intent, source_revision="3f68c145") assert exc_info.value.diagnostic.code == "unsupported-task-engine" - assert exc_info.value.diagnostic.details == { - "engine": "vllm", - "task": "entity-detection", - } + + +def test_compiler_rejects_unpinned_or_uncharacterized_factory_models() -> None: + """Factory plans close both checkpoint provenance and plugin compatibility.""" + models, compiler = load_compiler_modules() + + def compile_model(model_id: str, revision: str | None): + return compiler.compile_intent( + models.InferenceIntent( + task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), + model=models.HuggingFaceModel(model_id=model_id, revision=revision), + engine=models.VllmEngine(factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), + placement=models.LocalProcessPlacement(), + access=models.DirectAccess(), + lifecycle=models.ManagedLifecycle(), + ), + source_revision="3f68c145", + ) + + with pytest.raises(compiler.CompilationError) as unpinned: + compile_model("nvidia/gliner-pii", None) + assert unpinned.value.diagnostic.code == "unpinned-model-revision" + + with pytest.raises(compiler.CompilationError) as unsupported: + compile_model("urchade/gliner_small-v2.1", "abcdef0123456789") + assert unsupported.value.diagnostic.code == "unsupported-model-engine" def test_compiler_rejects_unsupported_native_generation() -> None: diff --git a/tests/tools/test_vllm_factory.py b/tests/tools/test_vllm_factory.py index 31347a52..4414d7ef 100644 --- a/tests/tools/test_vllm_factory.py +++ b/tests/tools/test_vllm_factory.py @@ -17,18 +17,24 @@ REPO_ROOT = TOOLS_ROOT.parent -def test_local_models_group_pins_vllm_0_26() -> None: - """The characterized Python factory stays bound to the reviewed vLLM release.""" +def test_local_models_group_pins_vllm_and_external_factory_source() -> None: + """The runtime pins vLLM and the reviewed external factory source revision.""" project = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) - assert project["dependency-groups"]["local-models"] == ["vllm==0.26.0; sys_platform == 'linux'"] + assert project["dependency-groups"]["local-models"] == [ + "vllm==0.26.0; sys_platform == 'linux'", + ( + "vllm-factory[gliner] @ git+https://github.com/latenceainew/vllm-factory.git@" + "7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0; sys_platform == 'linux'" + ), + ] def load_factory_module(): - """Load the source-tree factory without packaging it.""" + """Load the source-tree runtime without packaging it.""" sys.path.insert(0, str(TOOLS_ROOT)) try: - return importlib.import_module("inference_service_compiler.vllm_factory") + return importlib.import_module("inference_service_compiler.vllm_runtime") finally: sys.path.pop(0) @@ -57,6 +63,8 @@ def test_parse_server_parameters_accepts_only_the_compiler_contract() -> None: "--max-model-len", "2048", "--enforce-eager", + "--vllm-factory-plugin", + "deberta_gliner", ] ) @@ -65,6 +73,7 @@ def test_parse_server_parameters_accepts_only_the_compiler_contract() -> None: assert parameters.tensor_parallel_size == 2 assert parameters.gpu_memory_utilization == 0.8 assert parameters.enforce_eager is True + assert parameters.vllm_factory_plugin == "deberta_gliner" def test_factory_constructs_vllm_frontend_and_async_engine_arguments() -> None: @@ -101,6 +110,28 @@ def test_factory_constructs_vllm_frontend_and_async_engine_arguments() -> None: assert [(module.name, module.path) for module in arguments.lora_modules] == [("privacy", "/models/privacy-adapter")] +def test_factory_constructs_pooling_server_for_external_gliner_plugin() -> None: + """Factory-backed detection uses vLLM pooling and the project's IOProcessor.""" + pytest.importorskip("vllm") + factory = load_factory_module() + parameters = factory.VllmServerParameters( + model="/tmp/prepared-gliner", + host="127.0.0.1", + port=8123, + served_model_name="nvidia/gliner-pii", + vllm_factory_plugin="deberta_gliner", + ) + + arguments = factory.build_server_arguments(parameters) + + assert arguments.runner == "pooling" + assert arguments.io_processor_plugin == "deberta_gliner_io" + assert arguments.trust_remote_code is True + assert arguments.enable_prefix_caching is False + assert arguments.enable_chunked_prefill is False + assert arguments.middleware == [factory.ANONYMIZER_CHAT_MIDDLEWARE] + + def test_run_server_uses_the_vllm_0_26_lifecycle_boundary() -> None: """The process runner imports and invokes vLLM 0.26's relocated setup API.""" pytest.importorskip("vllm") @@ -115,7 +146,10 @@ def test_run_server_uses_the_vllm_0_26_lifecycle_boundary() -> None: with ( mock.patch.object(factory, "parse_server_parameters", return_value=mock.sentinel.parameters), - mock.patch.object(factory, "build_server_arguments", return_value=arguments), + mock.patch.object( + factory, "prepare_runtime_environment", return_value=mock.sentinel.prepared + ) as prepare_environment, + mock.patch.object(factory, "build_server_arguments", return_value=arguments) as build_arguments, mock.patch.object(api_utils, "cli_env_setup") as cli_env_setup, mock.patch.object(api_server, "run_server", new=run_vllm_server), mock.patch.object(uvloop, "run") as uvloop_run, @@ -123,6 +157,8 @@ def test_run_server_uses_the_vllm_0_26_lifecycle_boundary() -> None: factory.run_server(["model", "--host", "127.0.0.1", "--port", "8000"]) cli_env_setup.assert_called_once_with() + prepare_environment.assert_called_once_with(mock.sentinel.parameters) + build_arguments.assert_called_once_with(mock.sentinel.prepared) run_vllm_server.assert_called_once_with(arguments) uvloop_run.assert_called_once_with(coroutine) @@ -143,11 +179,32 @@ def test_factory_exposes_interpreter_tools_on_path() -> None: def test_factory_avoids_flashinfer_jit_without_overriding_operator_choice() -> None: """The wheel-only runtime does not require a host CUDA compiler by default.""" factory = load_factory_module() + parameters = factory.VllmServerParameters(model="model", host="127.0.0.1", port=8000) with mock.patch.dict(os.environ, {}, clear=True): - factory.prepare_runtime_environment() + assert factory.prepare_runtime_environment(parameters) == parameters assert os.environ["VLLM_USE_FLASHINFER_SAMPLER"] == "0" with mock.patch.dict(os.environ, {"VLLM_USE_FLASHINFER_SAMPLER": "1"}, clear=True): - factory.prepare_runtime_environment() + assert factory.prepare_runtime_environment(parameters) == parameters assert os.environ["VLLM_USE_FLASHINFER_SAMPLER"] == "1" + + +def test_factory_selects_model_and_io_plugins_together() -> None: + """vLLM's shared plugin allowlist retains both factory entry-point groups.""" + factory = load_factory_module() + parameters = factory.VllmServerParameters( + model="nvidia/gliner-pii", + revision="bd23e8ef", + host="127.0.0.1", + port=8000, + vllm_factory_plugin="deberta_gliner", + ) + + with ( + mock.patch.object(factory, "prepare_model", return_value="/tmp/prepared"), + mock.patch.dict(os.environ, {}, clear=True), + ): + factory.prepare_runtime_environment(parameters) + + assert os.environ["VLLM_PLUGINS"] == "deberta_gliner,deberta_gliner_io" diff --git a/tests/tools/test_vllm_factory_adapter.py b/tests/tools/test_vllm_factory_adapter.py new file mode 100644 index 00000000..be91ac65 --- /dev/null +++ b/tests/tools/test_vllm_factory_adapter.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Behavior tests for the vLLM Factory detector protocol adapter.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path + +TOOLS_ROOT = Path(__file__).resolve().parents[2] / "tools" + + +def load_adapter(): + """Load the source-tree adapter without installing it as a package.""" + sys.path.insert(0, str(TOOLS_ROOT)) + try: + return importlib.import_module("inference_service_compiler.vllm_factory_adapter") + finally: + sys.path.pop(0) + + +def test_parse_detection_request_preserves_anonymizer_options() -> None: + """The adapter accepts the detector extras emitted by DataDesigner.""" + adapter = load_adapter() + + request = adapter.parse_detection_request( + { + "model": "nvidia/gliner-pii", + "messages": [{"role": "user", "content": "Ada Lovelace"}], + "labels": ["person", "email"], + "threshold": 0.42, + "chunk_length": 256, + "overlap": 64, + "flat_ner": True, + } + ) + + assert request.text == "Ada Lovelace" + assert request.labels == ("person", "email") + assert request.threshold == 0.42 + assert request.chunk_length == 256 + assert request.overlap == 64 + assert request.flat_ner is True + + +def test_parse_detection_request_accepts_label_free_health_check() -> None: + """DataDesigner's generic model health check receives a valid empty result.""" + adapter = load_adapter() + + request = adapter.parse_detection_request( + { + "model": "nvidia/gliner-pii", + "messages": [{"role": "user", "content": "health check"}], + } + ) + + assert request.labels == () + + +def test_merge_gliner_entities_restores_offsets_and_deduplicates_overlap() -> None: + """Chunk-relative factory spans become stable document offsets.""" + adapter = load_adapter() + chunks = [("Alice met Bob", 0), ("Bob at NVIDIA", 10)] + results = [ + [ + {"text": "Alice", "label": "person", "start": 0, "end": 5, "score": 0.9}, + {"text": "Bob", "label": "person", "start": 10, "end": 13, "score": 0.8}, + ], + [ + {"text": "Bob", "label": "person", "start": 0, "end": 3, "score": 0.95}, + {"text": "NVIDIA", "label": "company", "start": 7, "end": 13, "score": 0.88}, + ], + ] + + entities = adapter.merge_entities( + plugin="deberta_gliner", + chunks=chunks, + results=results, + flat_ner=True, + ) + + assert [entity.as_dict() for entity in entities] == [ + {"text": "Alice", "label": "person", "start": 0, "end": 5, "score": 0.9}, + {"text": "Bob", "label": "person", "start": 10, "end": 13, "score": 0.95}, + {"text": "NVIDIA", "label": "company", "start": 17, "end": 23, "score": 0.88}, + ] + + +def test_merge_gliner2_entities_normalizes_confidence_and_spans() -> None: + """GLiNER2's schema result becomes the detector's flat entity list.""" + adapter = load_adapter() + + entities = adapter.merge_entities( + plugin="deberta_gliner2", + chunks=[("Email alice@example.com", 0)], + results=[ + { + "entities": { + "email": [ + { + "text": "alice@example.com", + "start": 6, + "end": 23, + "confidence": 0.97, + } + ] + } + } + ], + flat_ner=False, + ) + + assert [entity.as_dict() for entity in entities] == [ + { + "text": "alice@example.com", + "label": "email", + "start": 6, + "end": 23, + "score": 0.97, + } + ] + + +def test_split_text_matches_native_character_overlap_contract() -> None: + """The adapter keeps the characterized character-based chunk semantics.""" + adapter = load_adapter() + + assert adapter.split_text("abcdefghij", chunk_length=6, overlap=2) == [ + ("abcdef", 0), + ("efghij", 4), + ] diff --git a/tests/tools/test_vllm_factory_integration.py b/tests/tools/test_vllm_factory_integration.py new file mode 100644 index 00000000..0ecc6787 --- /dev/null +++ b/tests/tools/test_vllm_factory_integration.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the pinned external vLLM Factory preparation boundary.""" + +from __future__ import annotations + +import importlib +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +TOOLS_ROOT = Path(__file__).resolve().parents[2] / "tools" + + +def load_integration(): + """Load the source-tree integration without installing it as a package.""" + sys.path.insert(0, str(TOOLS_ROOT)) + try: + return importlib.import_module("inference_service_compiler.vllm_factory_integration") + finally: + sys.path.pop(0) + + +def test_prepare_model_injects_pinned_revision_into_upstream_python_api(tmp_path: Path) -> None: + """Anonymizer closes vLLM Factory's missing Hugging Face revision argument.""" + integration = load_integration() + download = mock.Mock(return_value="/cache/file") + list_files = mock.Mock(return_value=[]) + tokenizer = mock.Mock(return_value=mock.Mock()) + model_prep = SimpleNamespace( + hf_hub_download=download, + list_repo_files=list_files, + ) + + def prepare_model_for_vllm_if_needed(**kwargs): + model_prep.list_repo_files(kwargs["model_ref"]) + model_prep.hf_hub_download(repo_id=kwargs["model_ref"], filename="config.json") + transformers.AutoTokenizer.from_pretrained(kwargs["model_ref"]) + Path(kwargs["output_dir"]).mkdir(parents=True, exist_ok=True) + return kwargs["output_dir"] + + model_prep.prepare_model_for_vllm_if_needed = prepare_model_for_vllm_if_needed + transformers = SimpleNamespace(AutoTokenizer=SimpleNamespace(from_pretrained=tokenizer)) + + def import_module(name: str): + return model_prep if name == "forge.model_prep" else transformers + + with mock.patch.object(integration.importlib, "import_module", side_effect=import_module): + prepared = integration.prepare_model( + model_id="nvidia/gliner-pii", + revision="bd23e8ef", + plugin="deberta_gliner", + prepared_model_root=str(tmp_path), + ) + + assert Path(prepared).is_dir() + list_files.assert_called_once_with("nvidia/gliner-pii", revision="bd23e8ef") + download.assert_called_once_with( + repo_id="nvidia/gliner-pii", + filename="config.json", + revision="bd23e8ef", + ) + tokenizer.assert_called_once_with("nvidia/gliner-pii", revision="bd23e8ef") diff --git a/tools/inference_service_compiler/compiler.py b/tools/inference_service_compiler/compiler.py index bfbbae59..f29525a2 100644 --- a/tools/inference_service_compiler/compiler.py +++ b/tools/inference_service_compiler/compiler.py @@ -7,6 +7,7 @@ import hashlib import hmac import json +from typing import Never from pydantic import BaseModel @@ -32,6 +33,10 @@ SecretEnvironmentVariable, VllmEngine, ) +from inference_service_compiler.vllm_factory_integration import ( + VLLM_FACTORY_DEPENDENCY, + supports_model, +) VLLM_API_KEY_ENV = "VLLM_API_KEY" @@ -84,6 +89,7 @@ def compile_intent(intent: InferenceIntent, *, source_revision: str) -> RunPlan: required_capabilities=required, declared_capabilities=declared, compatibility_evidence=evidence, + dependencies=_plan_dependencies(intent), source_revision=source_revision, ) return plan.model_copy(update={"plan_digest": digest_plan(plan)}) @@ -201,7 +207,65 @@ def _compile_vllm( tuple[Capability, ...], tuple[CompatibilityEvidence, ...], ]: - if not isinstance(intent.task, Generation): + if isinstance(intent.task, EntityDetection): + factory = engine.factory + if factory is None: + _raise_unsupported_task_engine(intent.task.kind, engine.kind) + if intent.model.revision is None: + raise CompilationError( + CompilerDiagnostic( + code="unpinned-model-revision", + message="vLLM Factory entity detection requires a pinned model revision", + details={"model": intent.model.model_id}, + ) + ) + if not supports_model(factory.plugin, intent.model.model_id): + raise CompilationError( + CompilerDiagnostic( + code="unsupported-model-engine", + message=( + f"model {intent.model.model_id!r} is not characterized for " + f"vLLM Factory plugin {factory.plugin!r}" + ), + details={ + "model": intent.model.model_id, + "plugin": factory.plugin, + }, + ) + ) + if intent.model.adapter is not None: + raise CompilationError( + CompilerDiagnostic( + code="unsupported-model-adapter", + message="vLLM Factory entity detection does not support a model adapter", + details={"engine": engine.kind, "task": intent.task.kind}, + ) + ) + if isinstance(intent.placement, DockerPlacement): + raise CompilationError( + CompilerDiagnostic( + code="unsupported-engine-placement", + message="vLLM Factory entity detection is characterized only as a local process", + details={"engine": engine.kind, "placement": intent.placement.kind}, + ) + ) + command, runtime = _vllm_command(intent, engine) + return ( + command, + runtime, + intent.task.required_capabilities(), + ( + CompatibilityEvidence( + rule="vllm-factory-entity-detection-v1", + outcome="runtime-probe-required", + detail=( + "vLLM Factory supplies model preparation, pooling inference, and IO processing; " + "the Anonymizer adapter preserves dynamic labels, offsets, and scores" + ), + ), + ), + ) + if not isinstance(intent.task, Generation) or engine.factory is not None: _raise_unsupported_task_engine(intent.task.kind, engine.kind) command, runtime = _vllm_command(intent, engine) declared = ("chat-completions",) @@ -291,6 +355,15 @@ def _vllm_engine_arguments(intent: InferenceIntent, engine: VllmEngine) -> tuple arguments.extend(_literal_arguments("--max-model-len", str(engine.max_model_len))) if engine.eager: arguments.extend(_literal_arguments("--enforce-eager")) + if engine.factory is not None: + arguments.extend( + _literal_arguments( + "--vllm-factory-plugin", + engine.factory.plugin, + "--prepared-model-root", + engine.factory.prepared_model_root, + ) + ) if intent.model.adapter is not None: arguments.extend( _literal_arguments( @@ -317,7 +390,16 @@ def _literal_arguments(*values: str) -> tuple[CommandArgument, ...]: return tuple(LiteralArgument(value=value) for value in values) -def _raise_unsupported_task_engine(task: str, engine: str) -> None: +def _plan_dependencies(intent: InferenceIntent) -> tuple[str, ...]: + if isinstance(intent.engine, VllmEngine): + dependencies = ["vllm==0.26.0"] + if intent.engine.factory is not None: + dependencies.append(VLLM_FACTORY_DEPENDENCY) + return tuple(dependencies) + return () + + +def _raise_unsupported_task_engine(task: str, engine: str) -> Never: raise CompilationError( CompilerDiagnostic( code="unsupported-task-engine", diff --git a/tools/inference_service_compiler/models.py b/tools/inference_service_compiler/models.py index f75ee4de..6135337f 100644 --- a/tools/inference_service_compiler/models.py +++ b/tools/inference_service_compiler/models.py @@ -90,6 +90,13 @@ class NativeGlinerEngine(FrozenModel): log_format: Literal["plain", "json"] = "plain" +class VllmFactoryIntegration(FrozenModel): + """A supported vLLM Factory structured-prediction plugin.""" + + plugin: Literal["deberta_gliner", "deberta_gliner2"] + prepared_model_root: str = Field(default="/tmp/anonymizer-vllm-factory", min_length=1) + + class VllmEngine(FrozenModel): """vLLM's OpenAI-compatible server with bounded common options.""" @@ -101,6 +108,7 @@ class VllmEngine(FrozenModel): gpu_memory_utilization: float | None = Field(default=None, gt=0, le=1) max_model_len: int | None = Field(default=None, ge=1) eager: bool = False + factory: VllmFactoryIntegration | None = None EngineSpec = Annotated[NativeGlinerEngine | VllmEngine, Field(discriminator="kind")] diff --git a/tools/inference_service_compiler/vllm_factory_adapter.py b/tools/inference_service_compiler/vllm_factory_adapter.py new file mode 100644 index 00000000..ad1cba09 --- /dev/null +++ b/tools/inference_service_compiler/vllm_factory_adapter.py @@ -0,0 +1,334 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Translate Anonymizer detector requests into vLLM Factory pooling calls.""" + +from __future__ import annotations + +import asyncio +import importlib +import json +import math +import os +import uuid +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Any, cast + +DEFAULT_CHUNK_LENGTH = 384 +DEFAULT_OVERLAP = 128 + + +@dataclass(frozen=True) +class DetectionRequest: + """Validated Anonymizer detector request.""" + + model: str + text: str + labels: tuple[str, ...] + threshold: float + chunk_length: int + overlap: int + flat_ner: bool + + +@dataclass(frozen=True) +class Entity: + """One entity in Anonymizer's detector response shape.""" + + text: str + label: str + start: int + end: int + score: float + + def as_dict(self) -> dict[str, str | int | float]: + return { + "text": self.text, + "label": self.label, + "start": self.start, + "end": self.end, + "score": self.score, + } + + +async def anonymizer_chat_compatibility( + request: Any, + call_next: Callable[[Any], Awaitable[Any]], +) -> Any: + """Serve Anonymizer's chat contract through the active factory IOProcessor.""" + if request.url.path != "/v1/chat/completions": + return await call_next(request) + + responses = importlib.import_module("starlette.responses") + try: + detection = parse_detection_request(await request.json()) + plugin = os.environ["ANONYMIZER_VLLM_FACTORY_PLUGIN"] + entities: list[Entity] = [] + if detection.labels: + chunks = split_text(detection.text, detection.chunk_length, detection.overlap) + handler = request.app.state.serving_pooling + if handler is None: + raise RuntimeError("vLLM pooling handler is unavailable") + results = await asyncio.gather( + *( + invoke_pooling( + handler=handler, + model=detection.model, + plugin=plugin, + text=chunk, + labels=detection.labels, + threshold=detection.threshold, + flat_ner=detection.flat_ner, + ) + for chunk, _offset in chunks + ) + ) + entities = merge_entities( + plugin=plugin, + chunks=chunks, + results=results, + flat_ner=detection.flat_ner, + ) + except (KeyError, TypeError, ValueError, RuntimeError) as exc: + return responses.JSONResponse( + status_code=400, + content={"error": {"message": str(exc), "type": "invalid_request_error"}}, + ) + + content = json.dumps({"entities": [entity.as_dict() for entity in entities]}) + return responses.JSONResponse( + content={ + "id": f"chatcmpl-{uuid.uuid4().hex}", + "object": "chat.completion", + "model": detection.model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": content}, + "finish_reason": "stop", + } + ], + } + ) + + +def parse_detection_request(value: object) -> DetectionRequest: + """Validate the bounded chat-completions request used by Anonymizer.""" + body = require_mapping(value, "request body") + model = body.get("model") + if not isinstance(model, str) or not model: + raise ValueError("model must be a non-empty string") + labels_value = body.get("labels", []) + if not isinstance(labels_value, list) or not all(isinstance(label, str) and label for label in labels_value): + raise ValueError("labels must be a list of strings") + threshold = require_number(body.get("threshold", 0.3), "threshold") + if not 0 <= threshold <= 1: + raise ValueError("threshold must be between 0 and 1") + chunk_length = require_integer(body.get("chunk_length", DEFAULT_CHUNK_LENGTH), "chunk_length") + overlap = require_integer(body.get("overlap", DEFAULT_OVERLAP), "overlap") + if chunk_length < 1: + raise ValueError("chunk_length must be >= 1") + if overlap < 0 or overlap >= chunk_length: + raise ValueError("overlap must be >= 0 and less than chunk_length") + flat_ner = body.get("flat_ner", False) + if not isinstance(flat_ner, bool): + raise ValueError("flat_ner must be a boolean") + return DetectionRequest( + model=model, + text=extract_text(body.get("messages")), + labels=tuple(cast(list[str], labels_value)), + threshold=threshold, + chunk_length=chunk_length, + overlap=overlap, + flat_ner=flat_ner, + ) + + +def extract_text(messages: object) -> str: + """Extract text from the final user message.""" + if not isinstance(messages, list): + raise ValueError("messages must be a list") + if not messages: + return "" + message = require_mapping(messages[-1], "message") + content = message.get("content", "") + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for part in content: + text = require_mapping(part, "message part").get("text", "") + if not isinstance(text, str): + raise ValueError("message part text must be a string") + parts.append(text) + return "".join(parts) + raise ValueError("message content must be a string or list") + + +def split_text(text: str, chunk_length: int, overlap: int) -> list[tuple[str, int]]: + """Split text with the same character-offset contract as the native runtime.""" + if not text: + return [("", 0)] + chunks: list[tuple[str, int]] = [] + start = 0 + while start < len(text): + chunks.append((text[start : start + chunk_length], start)) + if start + chunk_length >= len(text): + break + start += chunk_length - overlap + return chunks + + +async def invoke_pooling( + *, + handler: Any, + model: str, + plugin: str, + text: str, + labels: tuple[str, ...], + threshold: float, + flat_ner: bool, +) -> object: + """Call vLLM's in-process pooling handler once for one text chunk.""" + protocol = importlib.import_module("vllm.entrypoints.pooling.pooling.protocol") + data: dict[str, object] = { + "text": text, + "labels": list(labels), + "threshold": threshold, + } + if plugin == "deberta_gliner": + data["flat_ner"] = flat_ner + elif plugin == "deberta_gliner2": + data["include_confidence"] = True + data["include_spans"] = True + else: + raise ValueError(f"unsupported vLLM Factory plugin {plugin!r}") + response = await handler(protocol.IOProcessorRequest(model=model, data=data), None) + if response.status_code != 200: + raise RuntimeError(f"vLLM Factory pooling request returned {response.status_code}") + body = getattr(response, "body", None) + if not isinstance(body, bytes): + raise RuntimeError("vLLM Factory pooling response did not contain a JSON body") + payload = require_mapping(json.loads(body), "pooling response") + if "data" not in payload: + raise ValueError("pooling response is missing data") + return payload["data"] + + +def merge_entities( + *, + plugin: str, + chunks: list[tuple[str, int]], + results: Sequence[object], + flat_ner: bool, +) -> list[Entity]: + """Normalize factory outputs, restore document offsets, and deduplicate overlap.""" + if len(chunks) != len(results): + raise ValueError("vLLM Factory returned an unexpected result count") + entities: list[Entity] = [] + for (chunk, offset), result in zip(chunks, results, strict=True): + normalized = normalize_gliner(result) if plugin == "deberta_gliner" else normalize_gliner2(result) + for entity in normalized: + if entity.start < 0 or entity.end < entity.start or entity.end > len(chunk): + raise ValueError("vLLM Factory returned an invalid entity span") + entities.append( + Entity( + text=entity.text, + label=entity.label, + start=entity.start + offset, + end=entity.end + offset, + score=entity.score, + ) + ) + if not flat_ner: + entities = remove_subset_entities(entities) + unique: dict[tuple[str, str, int, int], Entity] = {} + for entity in entities: + key = (entity.text.strip().casefold(), entity.label, entity.start, entity.end) + previous = unique.get(key) + if previous is None or entity.score > previous.score: + unique[key] = entity + return sorted(unique.values(), key=lambda item: (item.start, item.end, item.label)) + + +def normalize_gliner(value: object) -> list[Entity]: + """Normalize the vLLM Factory DeBERTa GLiNER IOProcessor output.""" + if not isinstance(value, list): + raise ValueError("GLiNER pooling data must be a list") + entities: list[Entity] = [] + for item in value: + record = require_mapping(item, "GLiNER entity") + entities.append( + Entity( + text=require_string(record.get("text"), "entity text"), + label=require_string(record.get("label"), "entity label"), + start=require_integer(record.get("start"), "entity start"), + end=require_integer(record.get("end"), "entity end"), + score=require_number(record.get("score"), "entity score"), + ) + ) + return entities + + +def normalize_gliner2(value: object) -> list[Entity]: + """Normalize the vLLM Factory DeBERTa GLiNER2 IOProcessor output.""" + payload = require_mapping(value, "GLiNER2 pooling data") + by_label = require_mapping(payload.get("entities"), "GLiNER2 entities") + entities: list[Entity] = [] + for label, records in by_label.items(): + if not isinstance(records, list): + raise ValueError("GLiNER2 entity values must be lists") + for item in records: + record = require_mapping(item, "GLiNER2 entity") + entities.append( + Entity( + text=require_string(record.get("text"), "entity text"), + label=label, + start=require_integer(record.get("start"), "entity start"), + end=require_integer(record.get("end"), "entity end"), + score=require_number(record.get("confidence"), "entity confidence"), + ) + ) + return entities + + +def remove_subset_entities(entities: list[Entity]) -> list[Entity]: + """Remove spans strictly contained by another detected span.""" + return [ + entity + for entity in entities + if not any( + other is not entity + and other.start <= entity.start + and other.end >= entity.end + and (other.start < entity.start or other.end > entity.end) + for other in entities + ) + ] + + +def require_mapping(value: object, name: str) -> Mapping[str, object]: + if not isinstance(value, Mapping) or any(not isinstance(key, str) for key in value): + raise ValueError(f"unexpected {name} shape") + return cast(Mapping[str, object], value) + + +def require_string(value: object, name: str) -> str: + if not isinstance(value, str): + raise ValueError(f"{name} must be a string") + return value + + +def require_integer(value: object, name: str) -> int: + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"{name} must be an integer") + return value + + +def require_number(value: object, name: str) -> float: + if isinstance(value, bool) or not isinstance(value, int | float): + raise ValueError(f"{name} must be a number") + number = float(value) + if not math.isfinite(number): + raise ValueError(f"{name} must be finite") + return number diff --git a/tools/inference_service_compiler/vllm_factory_integration.py b/tools/inference_service_compiler/vllm_factory_integration.py new file mode 100644 index 00000000..6fada61c --- /dev/null +++ b/tools/inference_service_compiler/vllm_factory_integration.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pinned integration with the external vLLM Factory project.""" + +from __future__ import annotations + +import importlib +import json +import re +from contextlib import contextmanager +from pathlib import Path +from typing import Any, Iterator + +VLLM_FACTORY_SOURCE_REVISION = "7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0" +VLLM_FACTORY_SOURCE_URL = "https://github.com/latenceainew/vllm-factory.git" +VLLM_FACTORY_DEPENDENCY = f"vllm-factory[gliner] @ git+{VLLM_FACTORY_SOURCE_URL}@{VLLM_FACTORY_SOURCE_REVISION}" + +PLUGIN_IO_PROCESSORS = { + "deberta_gliner": "deberta_gliner_io", + "deberta_gliner2": "deberta_gliner2_io", +} +CHARACTERIZED_MODELS = { + "deberta_gliner": frozenset({"nvidia/gliner-pii"}), + "deberta_gliner2": frozenset({"fastino/gliner2-privacy-filter-PII-multi"}), +} + + +def prepare_model( + *, + model_id: str, + revision: str | None, + plugin: str, + prepared_model_root: str, +) -> str: + """Prepare one pinned model through vLLM Factory's Python API.""" + if plugin not in PLUGIN_IO_PROCESSORS: + raise ValueError(f"unsupported vLLM Factory plugin {plugin!r}") + if revision is None: + raise ValueError("vLLM Factory models require a pinned Hugging Face revision") + + output = _prepared_model_path( + root=Path(prepared_model_root), + model_id=model_id, + revision=revision, + plugin=plugin, + ) + provenance_path = output / ".anonymizer-vllm-factory.json" + provenance = { + "model_id": model_id, + "model_revision": revision, + "plugin": plugin, + "vllm_factory_source": VLLM_FACTORY_SOURCE_URL, + "vllm_factory_revision": VLLM_FACTORY_SOURCE_REVISION, + } + force = not _matches_provenance(provenance_path, provenance) + + model_prep = importlib.import_module("forge.model_prep") + transformers = importlib.import_module("transformers") + with _pin_hugging_face_revision(model_prep, transformers, model_id, revision): + prepared = model_prep.prepare_model_for_vllm_if_needed( + model_ref=model_id, + plugin=plugin, + output_dir=str(output), + force=force, + ) + if prepared == model_id: + raise RuntimeError(f"vLLM Factory did not prepare GLiNER model {model_id!r}") + output.mkdir(parents=True, exist_ok=True) + provenance_path.write_text(json.dumps(provenance, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return str(output) + + +def io_processor_for(plugin: str) -> str: + """Resolve the vLLM IOProcessor entry point for a supported factory plugin.""" + try: + return PLUGIN_IO_PROCESSORS[plugin] + except KeyError as exc: + raise ValueError(f"unsupported vLLM Factory plugin {plugin!r}") from exc + + +def supports_model(plugin: str, model_id: str) -> bool: + """Return whether this source revision was characterized for the pair.""" + return model_id in CHARACTERIZED_MODELS.get(plugin, ()) + + +def _prepared_model_path(*, root: Path, model_id: str, revision: str, plugin: str) -> Path: + safe_model = re.sub(r"[^A-Za-z0-9_.-]+", "--", model_id).strip("-") + return root / safe_model / revision / plugin + + +def _matches_provenance(path: Path, expected: dict[str, str]) -> bool: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return False + return value == expected + + +@contextmanager +def _pin_hugging_face_revision( + model_prep: Any, + transformers: Any, + model_id: str, + revision: str, +) -> Iterator[None]: + """Supply the missing revision argument at vLLM Factory's hub boundary.""" + original_download = model_prep.hf_hub_download + original_list = model_prep.list_repo_files + original_tokenizer = transformers.AutoTokenizer.from_pretrained + + def pinned_download(*args: Any, **kwargs: Any) -> Any: + kwargs["revision"] = revision + return original_download(*args, **kwargs) + + def pinned_list(*args: Any, **kwargs: Any) -> Any: + kwargs["revision"] = revision + return original_list(*args, **kwargs) + + def pinned_tokenizer(source: str, *args: Any, **kwargs: Any) -> Any: + if source == model_id: + kwargs["revision"] = revision + return original_tokenizer(source, *args, **kwargs) + + model_prep.hf_hub_download = pinned_download + model_prep.list_repo_files = pinned_list + transformers.AutoTokenizer.from_pretrained = pinned_tokenizer + try: + yield + finally: + model_prep.hf_hub_download = original_download + model_prep.list_repo_files = original_list + transformers.AutoTokenizer.from_pretrained = original_tokenizer diff --git a/tools/inference_service_compiler/vllm_factory.py b/tools/inference_service_compiler/vllm_runtime.py similarity index 67% rename from tools/inference_service_compiler/vllm_factory.py rename to tools/inference_service_compiler/vllm_runtime.py index e0dfcd1e..7924c380 100644 --- a/tools/inference_service_compiler/vllm_factory.py +++ b/tools/inference_service_compiler/vllm_runtime.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Programmatic construction of vLLM's OpenAI-compatible server.""" +"""Programmatic construction of stock vLLM and vLLM Factory servers.""" from __future__ import annotations @@ -10,9 +10,16 @@ import sys from argparse import Namespace from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path +from inference_service_compiler.vllm_factory_integration import ( + io_processor_for, + prepare_model, +) + +ANONYMIZER_CHAT_MIDDLEWARE = "inference_service_compiler.vllm_factory_adapter.anonymizer_chat_compatibility" + @dataclass(frozen=True) class VllmServerParameters: @@ -29,11 +36,13 @@ class VllmServerParameters: max_model_len: int | None = None enforce_eager: bool = False lora_module: str | None = None + vllm_factory_plugin: str | None = None + prepared_model_root: str = "/tmp/anonymizer-vllm-factory" def parse_server_parameters(argv: Sequence[str]) -> VllmServerParameters: """Parse the compiler's bounded process contract without using vLLM's CLI.""" - parser = argparse.ArgumentParser(description="Anonymizer-managed vLLM OpenAI server") + parser = argparse.ArgumentParser(description="Anonymizer-managed vLLM server") parser.add_argument("model") parser.add_argument("--host", required=True) parser.add_argument("--port", required=True, type=int) @@ -46,6 +55,11 @@ def parse_server_parameters(argv: Sequence[str]) -> VllmServerParameters: parser.add_argument("--enforce-eager", action="store_true") parser.add_argument("--enable-lora", action="store_true") parser.add_argument("--lora-modules") + parser.add_argument( + "--vllm-factory-plugin", + choices=("deberta_gliner", "deberta_gliner2"), + ) + parser.add_argument("--prepared-model-root", default="/tmp/anonymizer-vllm-factory") parsed = parser.parse_args(list(argv)) if parsed.enable_lora != (parsed.lora_modules is not None): parser.error("--enable-lora and --lora-modules must be used together") @@ -61,6 +75,8 @@ def parse_server_parameters(argv: Sequence[str]) -> VllmServerParameters: max_model_len=parsed.max_model_len, enforce_eager=parsed.enforce_eager, lora_module=parsed.lora_modules, + vllm_factory_plugin=parsed.vllm_factory_plugin, + prepared_model_root=parsed.prepared_model_root, ) @@ -77,6 +93,7 @@ def build_server_arguments(parameters: VllmServerParameters) -> Namespace: raise ValueError("LoRA module must use the form NAME=PATH") lora_modules = [model_protocol.LoRAModulePath(name=name, path=path)] + factory_plugin = parameters.vllm_factory_plugin engine = arg_utils.AsyncEngineArgs( model=parameters.model, revision=parameters.revision, @@ -86,6 +103,12 @@ def build_server_arguments(parameters: VllmServerParameters) -> Namespace: gpu_memory_utilization=parameters.gpu_memory_utilization or 0.9, enforce_eager=parameters.enforce_eager, enable_lora=lora_modules is not None, + runner="pooling" if factory_plugin is not None else "auto", + trust_remote_code=factory_plugin is not None, + dtype="bfloat16" if factory_plugin is not None else "auto", + enable_prefix_caching=False if factory_plugin is not None else None, + enable_chunked_prefill=False if factory_plugin is not None else None, + io_processor_plugin=io_processor_for(factory_plugin) if factory_plugin is not None else None, ) if parameters.max_model_len is not None: engine.max_model_len = parameters.max_model_len @@ -93,6 +116,7 @@ def build_server_arguments(parameters: VllmServerParameters) -> Namespace: host=parameters.host, port=parameters.port, lora_modules=lora_modules, + middleware=[ANONYMIZER_CHAT_MIDDLEWARE] if factory_plugin is not None else [], ) values = vars(engine) | vars(frontend) values.update( @@ -115,19 +139,38 @@ def expose_interpreter_tools() -> None: os.environ["PATH"] = os.pathsep.join([interpreter_bin, *path_entries]) -def prepare_runtime_environment() -> None: - """Prepare a wheel-only vLLM runtime without requiring a host CUDA compiler.""" +def prepare_runtime_environment(parameters: VllmServerParameters) -> VllmServerParameters: + """Prepare the selected stock vLLM or vLLM Factory runtime.""" expose_interpreter_tools() os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0") + plugin = parameters.vllm_factory_plugin + if plugin is None: + return parameters + os.environ["VLLM_PLUGINS"] = f"{plugin},{io_processor_for(plugin)}" + os.environ["ANONYMIZER_VLLM_FACTORY_PLUGIN"] = plugin + prepared_model = prepare_model( + model_id=parameters.model, + revision=parameters.revision, + plugin=plugin, + prepared_model_root=parameters.prepared_model_root, + ) + public_name = parameters.served_model_name or parameters.model + return replace( + parameters, + model=prepared_model, + revision=None, + tokenizer_revision=None, + served_model_name=public_name, + ) def run_server(argv: Sequence[str]) -> None: - """Construct and run vLLM's Python-owned OpenAI server lifecycle.""" - prepare_runtime_environment() + """Construct and run vLLM's Python-owned server lifecycle.""" + parameters = prepare_runtime_environment(parse_server_parameters(argv)) uvloop = importlib.import_module("uvloop") api_server = importlib.import_module("vllm.entrypoints.openai.api_server") api_utils = importlib.import_module("vllm.entrypoints.serve.utils.api_utils") api_utils.cli_env_setup() - arguments = build_server_arguments(parse_server_parameters(argv)) + arguments = build_server_arguments(parameters) uvloop.run(api_server.run_server(arguments)) diff --git a/tools/inference_service_compiler/vllm_server.py b/tools/inference_service_compiler/vllm_server.py index 975c391f..b5ddb5b5 100644 --- a/tools/inference_service_compiler/vllm_server.py +++ b/tools/inference_service_compiler/vllm_server.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Internal process entry point for the programmatic vLLM server factory.""" +"""Internal process entry point for the programmatic vLLM runtime.""" from __future__ import annotations @@ -10,7 +10,7 @@ TOOLS_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(TOOLS_ROOT)) -from inference_service_compiler.vllm_factory import run_server # noqa: E402 +from inference_service_compiler.vllm_runtime import run_server # noqa: E402 if __name__ == "__main__": run_server(sys.argv[1:]) diff --git a/tools/inference_service_profiles/gliner2.toml b/tools/inference_service_profiles/gliner2.toml index 95960786..6376e6d9 100644 --- a/tools/inference_service_profiles/gliner2.toml +++ b/tools/inference_service_profiles/gliner2.toml @@ -12,9 +12,14 @@ model_id = "fastino/gliner2-privacy-filter-PII-multi" revision = "59894c087cb2923b01f337d4ee72f6ff84d5bdd6" [engine] -kind = "native-gliner" -family = "gliner2" -device = "auto" +kind = "vllm" +python_executable = ".venv/bin/python" +gpu_memory_utilization = 0.85 +max_model_len = 512 + +[engine.factory] +plugin = "deberta_gliner2" +prepared_model_root = "/tmp/anonymizer-vllm-factory" [placement] kind = "local-process" diff --git a/tools/inference_service_profiles/nvidia-gliner.toml b/tools/inference_service_profiles/nvidia-gliner.toml index 12262f7a..2a0de58c 100644 --- a/tools/inference_service_profiles/nvidia-gliner.toml +++ b/tools/inference_service_profiles/nvidia-gliner.toml @@ -12,9 +12,14 @@ model_id = "nvidia/gliner-pii" revision = "bd23e8ef4425fd04e34c5204ab49ffaa706eae79" [engine] -kind = "native-gliner" -family = "nvidia-gliner" -device = "auto" +kind = "vllm" +python_executable = ".venv/bin/python" +gpu_memory_utilization = 0.85 +max_model_len = 512 + +[engine.factory] +plugin = "deberta_gliner" +prepared_model_root = "/tmp/anonymizer-vllm-factory" [placement] kind = "local-process" diff --git a/uv.lock b/uv.lock index 8e87c400..3d6e1ec6 100644 --- a/uv.lock +++ b/uv.lock @@ -1550,6 +1550,14 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f2/8f/b101913cb2b3687654f56681cfe9836d447526be663c149966470ef70531/flashinfer_python-0.6.14-py3-none-any.whl", hash = "sha256:d124369346a3d48eac67e31c42f7a3c813bcc0abc10e2e36db413b7b3dfd97df", size = 14574383, upload-time = "2026-07-02T00:22:48.413Z" }, ] +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl", hash = "sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4", size = 26661, upload-time = "2025-12-19T23:16:13.622Z" }, +] + [[package]] name = "fqdn" version = "1.5.1" @@ -1714,6 +1722,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/7a/1c6e3562dfd8950adbb11ffbc65d21e7c89d01a6e4f137fa981056de25c5/gitpython-3.1.50-py3-none-any.whl", hash = "sha256:d352abe2908d07355014abdd21ddf798c2a961469239afec4962e9da884858f9", size = 212507, upload-time = "2026-05-06T04:01:23.799Z" }, ] +[[package]] +name = "gliner" +version = "0.2.28" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "huggingface-hub" }, + { name = "onnxruntime" }, + { name = "sentencepiece" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "transformers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f4/b7/0f3e24ff0b8c1c95121532a44e09b5bb87bd771c6bed0d387d51480645c5/gliner-0.2.28.tar.gz", hash = "sha256:b1637afb5cf4235fc871f1e21498831775b7bd19cefbda6dc5fe08ee88cb07a0", size = 263394, upload-time = "2026-07-24T14:03:49.833Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/24/cf6a9eb70bd8eb78a74f90104180c38ae0f93bc213fdd3224ea93c2fe74a/gliner-0.2.28-py3-none-any.whl", hash = "sha256:734e333ebf8a48c135aac5c05599f51051037513030127cbaf676ae71ca501c5", size = 245603, upload-time = "2026-07-24T14:03:48.337Z" }, +] + [[package]] name = "googleapis-common-protos" version = "1.75.1" @@ -3447,6 +3472,7 @@ docs = [ ] local-models = [ { name = "vllm", marker = "sys_platform == 'linux'" }, + { name = "vllm-factory", extra = ["gliner"], marker = "sys_platform == 'linux'" }, ] measurement = [ { name = "wandb", extra = ["workspaces"] }, @@ -3490,7 +3516,10 @@ docs = [ { name = "mkdocs-material" }, { name = "mkdocstrings", extras = ["python"] }, ] -local-models = [{ name = "vllm", marker = "sys_platform == 'linux'", specifier = "==0.26.0" }] +local-models = [ + { name = "vllm", marker = "sys_platform == 'linux'", specifier = "==0.26.0" }, + { name = "vllm-factory", extras = ["gliner"], marker = "sys_platform == 'linux'", git = "https://github.com/latenceainew/vllm-factory.git?rev=7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0" }, +] measurement = [{ name = "wandb", extras = ["workspaces"], specifier = ">=0.19,<1" }] notebooks = [ { name = "datasets", specifier = ">=4.0.0,<6" }, @@ -4128,6 +4157,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/7b/c1b96f13ef89bdf2a8c2f326a97bed89699271990d7c8624fda3fedc6e61/nvtx-0.2.15-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58653bf6fd8453947b9e5153da2ad7aeb0ceafa030de7f133efb3eada5da7ca7", size = 790247, upload-time = "2026-03-18T10:11:39.124Z" }, ] +[[package]] +name = "onnxruntime" +version = "1.28.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flatbuffers" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "packaging" }, + { name = "protobuf" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/ea/97/b7ce1bc8bb6048b5fe9129f55d6506dc19499068ef2e0a0af1ae3c8aa4e7/onnxruntime-1.28.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d66f9ceb29909c70839e4e4fb3435c7b490050d8f162bd5f3aba4ca01ee517f", size = 17039880, upload-time = "2026-07-25T01:21:37.538Z" }, + { url = "https://files.pythonhosted.org/packages/f3/17/4e5ecd8764f87573c495d834ce79e61ecca47f7a01d1e444a606e570edcb/onnxruntime-1.28.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a166b78ee04f3a37fa1ef82034b6a3ce96d9684e582d4d30b296de83e9998bb5", size = 19193162, upload-time = "2026-07-25T01:21:59.151Z" }, + { url = "https://files.pythonhosted.org/packages/28/5b/1d77e62097fdbe07e2dc827f389b1c4c0c275f6fab0369a8f46d2461af27/onnxruntime-1.28.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e81a23df16e7acb9d51b06d30cc098e49315ef9180f97bc2221d167b4b04d9c", size = 17050628, upload-time = "2026-07-25T01:21:40.481Z" }, + { url = "https://files.pythonhosted.org/packages/95/df/5486ab03e9be288d5268867054c8b04bebcf95bfd12e801c05cc67703dab/onnxruntime-1.28.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0a83bdb70d143cede762b677789bf2a7acca54b3fb82565601d5c30695aa933c", size = 19214257, upload-time = "2026-07-25T01:22:01.695Z" }, + { url = "https://files.pythonhosted.org/packages/c0/23/b46045c3bf67a9cf54c12f5df0f018a422c65fbb9d6072b10071bebfaae2/onnxruntime-1.28.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f649dd6f6452d12a8059888aa489fe519e062e18793dac72b9efa0f9fdb64135", size = 17049339, upload-time = "2026-07-25T01:21:43.005Z" }, + { url = "https://files.pythonhosted.org/packages/78/b6/8c5396e7894e77c5a7d1e026f3acb9dd39c4b5644e412e37a0055eaa3bc5/onnxruntime-1.28.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:54fa221d669282bd8f582708ce4c96010a7e9fb0661f9006b37fe2fedafb73fe", size = 19214329, upload-time = "2026-07-25T01:22:04.133Z" }, + { url = "https://files.pythonhosted.org/packages/47/28/810314fa88647af9f4cdaf438a30ad1cfebebb53ded55499232d7a0094e6/onnxruntime-1.28.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac301f53b1930402fc46c368e268acfed02f3207272aaff05070d7e09f96f031", size = 17057307, upload-time = "2026-07-25T01:21:45.492Z" }, + { url = "https://files.pythonhosted.org/packages/3d/cc/9e9f193cc0f29f263a8f09ec08487aed6c96ee856d5fd77da32a425c1949/onnxruntime-1.28.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7f022a1103cae591c75fc4565589a515f2ddd14a6ac8e8a05812dfeda142e28", size = 19222954, upload-time = "2026-07-25T01:22:06.952Z" }, + { url = "https://files.pythonhosted.org/packages/3f/e9/139180b4dd810329aaa42c238b4e6383c906202d98609ae29d66eb7c32b1/onnxruntime-1.28.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc2565e487b4896fb988d6383577d875d958e071fc5f6c3550bd5d02ae98264b", size = 17051950, upload-time = "2026-07-25T01:21:48.606Z" }, + { url = "https://files.pythonhosted.org/packages/03/88/9432428273356ad3c8aa01f52c1b3e7f53c4c0192748f41ad983872b436b/onnxruntime-1.28.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6afdc83f1317c136e92fc29f5ee9f058de59d87c0b22cee3fdbfbaa0ccc2098a", size = 19214924, upload-time = "2026-07-25T01:22:09.727Z" }, + { url = "https://files.pythonhosted.org/packages/0d/83/00e606bc25c756d76a267370c39b7516ad52f9cf134d7ff2bff8b6108bc4/onnxruntime-1.28.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e562d6e36a749f6764481c0ddb0f2af3d0b5a3c164291361d08803c557f369af", size = 17055518, upload-time = "2026-07-25T01:21:51.08Z" }, + { url = "https://files.pythonhosted.org/packages/94/a9/68707e1ce345cbdbcd4df65932ebc82a673e917d63eda0007ebcff948691/onnxruntime-1.28.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f6e92367ddce1e4d33cf295024f40192be6c6171a09208f515ba169ced06c8e", size = 19222976, upload-time = "2026-07-25T01:22:12.474Z" }, +] + [[package]] name = "openai" version = "3.0.0" @@ -6610,7 +6665,7 @@ wheels = [ [[package]] name = "transformers" -version = "5.15.0" +version = "5.13.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "huggingface-hub" }, @@ -6624,9 +6679,9 @@ dependencies = [ { name = "tqdm" }, { name = "typer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a6/3f/d89353267d511e18f137dfd7769d07837350c11b88408ce1dfe2e93e56c7/transformers-5.15.0.tar.gz", hash = "sha256:bbf98f57b2ddd7c4ecbccfa2c0069017aa6fd01cc204bd50cbc0eeadcf2a13b8", size = 9377983, upload-time = "2026-08-10T10:27:23.261Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/f7/418169401560cec2b61512e6bf37b0cfb4c8e27700cfa868a1de073cb65d/transformers-5.13.1.tar.gz", hash = "sha256:1e2452d6778a7482158df5d5dacf6bf775d5b2fdcfce33caaf7f6b0e5f3e3397", size = 9196891, upload-time = "2026-07-11T09:15:50.845Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d8/43/81355710a4c84e9420e11a86d41a5364deb561f2ef36dfdf254a07371bbb/transformers-5.15.0-py3-none-any.whl", hash = "sha256:d7f007736f67749ae9490c4f8cb5d30b452ae2d68c8675e50ba8d63ea7feb107", size = 11749280, upload-time = "2026-08-10T10:27:20.416Z" }, + { url = "https://files.pythonhosted.org/packages/29/47/54eacf96b5c835bbd6ca631aa2740e7705ed63d9e3a8afd2d2cc6d09cae5/transformers-5.13.1-py3-none-any.whl", hash = "sha256:53f0ea8aa397e29244c2377ba981bcaf0c87adcf44fbdd447ef6306522afcacd", size = 11503977, upload-time = "2026-07-11T09:15:46.801Z" }, ] [[package]] @@ -6906,6 +6961,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/96/86edd288415aafc2952bbb969b5ef4e8c58e5525185b60320730276921e6/vllm-0.26.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:adb1e4c9b46d0dfdb094121ae5aad670a42412dd813ed4e5db069ed6a15006de", size = 303698761, upload-time = "2026-07-25T10:40:32.107Z" }, ] +[[package]] +name = "vllm-factory" +version = "0.2.2" +source = { git = "https://github.com/latenceainew/vllm-factory.git?rev=7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0#7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0" } +dependencies = [ + { name = "aiohttp" }, + { name = "huggingface-hub" }, + { name = "pillow" }, + { name = "requests" }, + { name = "sentencepiece" }, + { name = "torch" }, + { name = "transformers" }, +] + +[package.optional-dependencies] +gliner = [ + { name = "gliner" }, +] + [[package]] name = "wandb" version = "0.27.2" From 6e2cb78cf26d708fd9f159bb4f43d3cb33b707ae Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Tue, 11 Aug 2026 23:30:46 +0000 Subject: [PATCH 07/28] feat(dev): support vLLM 0.27.1 Signed-off-by: Aaron Gonzales --- docs/concepts/inference-services.md | 38 ++- docs/concepts/self-hosting-gliner.md | 9 +- pyproject.toml | 7 +- skills/anonymizer/SKILL.md | 2 +- tests/tools/test_inference_service.py | 19 +- tests/tools/test_vllm_factory.py | 90 +++++- tools/inference_service_compiler/compiler.py | 24 +- tools/inference_service_compiler/models.py | 7 + .../vllm_runtime.py | 102 ++++++- .../nemotron-3.5-lightning.toml | 37 +++ uv.lock | 257 ++++++++++-------- 11 files changed, 455 insertions(+), 137 deletions(-) create mode 100644 tools/inference_service_profiles/nemotron-3.5-lightning.toml diff --git a/docs/concepts/inference-services.md b/docs/concepts/inference-services.md index 601e2336..07a0fd70 100644 --- a/docs/concepts/inference-services.md +++ b/docs/concepts/inference-services.md @@ -47,12 +47,14 @@ The pinned NVIDIA GLiNER and GLiNER2 profiles under group on a Linux GPU host: ```bash -uv sync --group dev --group local-models +uv sync --python 3.12 --group dev --group local-models python -m vllm_factory.compat.doctor nvidia-smi ``` -The dependency group pins vLLM 0.26.0 and an exact vLLM Factory source commit. +The dependency group pins vLLM 0.27.1 and an exact vLLM Factory source commit. +Local vLLM 0.27.1 serving requires Python 3.12 or later. The group also aligns +the CUDA 13.0 compiler wheels used by FlashInfer's Mamba kernel JIT. The compiled plan records both dependencies. The runtime calls vLLM Factory's model-preparation Python API with the profile's pinned Hugging Face revision, loads its GLiNER model plugin and IOProcessor, then constructs the vLLM server @@ -113,11 +115,11 @@ but does not use vLLM Factory's scheduler or IOProcessor plugins. On a Linux GPU host, install the optional source-tree dependency group: ```bash -uv sync --group dev --group local-models +uv sync --python 3.12 --group dev --group local-models nvidia-smi ``` -The `local-models` group pins vLLM 0.26.0. The local generation plan starts +The `local-models` group pins vLLM 0.27.1. The local generation plan starts `tools/inference_service_compiler/vllm_server.py`, which constructs vLLM's frontend and async engine through its Python API. It does not invoke `vllm serve` or inherit vLLM's full CLI surface. @@ -153,6 +155,32 @@ name = "privacy" The compiler renders the corresponding vLLM `--lora-modules` arguments. +### Nemotron 3.5 Lightning + +`tools/inference_service_profiles/nemotron-3.5-lightning.toml` pins +`nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16` and its immutable +Hugging Face revision. The profile uses the model's recommended FlashInfer +Mamba backend, float16 Mamba cache, stochastic cache rounding, asynchronous +scheduling, and prefix caching. It bounds context to 8,192 tokens and +concurrency to 16 so the BF16 model can share one 80 GB GPU with the GLiNER +profile. + +```bash +uv run tools/inference_service.py compile \ + --profile tools/inference_service_profiles/nemotron-3.5-lightning.toml \ + --source-revision 3f68c145 \ + --output lightning-plan.json + +uv run tools/inference_service.py launch \ + --plan lightning-plan.json \ + --output lightning-launch.json +``` + +Lightning enables reasoning by default. For Anonymizer's structured LLM roles, +set `inference_parameters.extra_body.chat_template_kwargs.enable_thinking` to +`false`. Keep `max_tokens` within the profile's context bound; 1,024 tokens is +enough for the detection and replace-evaluation schemas. + ## Docker vLLM generation Change the placement to Docker to use vLLM's official OpenAI-compatible image: @@ -162,7 +190,7 @@ Change the placement to Docker to use vLLM's official OpenAI-compatible image: kind = "docker" host = "127.0.0.1" port = 8000 -image = "vllm/vllm-openai:v0.26.0" +image = "vllm/vllm-openai:v0.27.1" runtime = "docker" gpus = "all" hugging_face_cache = "/home/user/.cache/huggingface" diff --git a/docs/concepts/self-hosting-gliner.md b/docs/concepts/self-hosting-gliner.md index 2dd60d2b..a92aea26 100644 --- a/docs/concepts/self-hosting-gliner.md +++ b/docs/concepts/self-hosting-gliner.md @@ -7,7 +7,7 @@ By default, Anonymizer's entity detection stage calls the hosted `nvidia/gliner- The default NVIDIA GLiNER model is small enough to share a GPU with a local LLM. The optional GLiNER2 PII model is also fully local. The pinned reference -profiles serve both models through vLLM 0.26 and the external +profiles serve both models through vLLM 0.27.1 and the external [vLLM Factory](https://github.com/latenceainew/vllm-factory) project. A native CPU, MPS, or GPU fallback remains available for custom profiles. @@ -106,13 +106,14 @@ Install [uv](https://docs.astral.sh/uv/) and sync the local model group on a Linux GPU host: ```bash -uv sync --group dev --group local-models +uv sync --python 3.12 --group dev --group local-models python -m vllm_factory.compat.doctor ``` -The group pins `vllm==0.26.0` and vLLM Factory at the exact source revision +The group pins `vllm==0.27.1` and vLLM Factory at the exact source revision recorded in `pyproject.toml` and `uv.lock`. The doctor must report the general -plugins group, IOProcessor plugins group, and native IO mode. +plugins group, IOProcessor plugins group, and native IO mode. Local vLLM 0.27.1 +serving requires Python 3.12 or later. On first launch, the selected public checkpoint is downloaded from Hugging Face and cached under `~/.cache/huggingface/`. The integration supplies the TOML diff --git a/pyproject.toml b/pyproject.toml index 9af1203e..5d4f4475 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,8 +54,11 @@ notebooks = [ "pillow>=12.0.0,<13", ] local-models = [ - "vllm==0.26.0; sys_platform == 'linux'", - "vllm-factory[gliner] @ git+https://github.com/latenceainew/vllm-factory.git@7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0; sys_platform == 'linux'", + "vllm==0.27.1; sys_platform == 'linux' and python_version >= '3.12'", + "vllm-factory[gliner] @ git+https://github.com/latenceainew/vllm-factory.git@7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0; sys_platform == 'linux' and python_version >= '3.12'", + "nvidia-cuda-nvcc==13.0.88; sys_platform == 'linux' and python_version >= '3.12'", + "nvidia-cuda-crt==13.0.88; sys_platform == 'linux' and python_version >= '3.12'", + "nvidia-nvvm==13.0.88; sys_platform == 'linux' and python_version >= '3.12'", ] [build-system] diff --git a/skills/anonymizer/SKILL.md b/skills/anonymizer/SKILL.md index b072b523..863d49d5 100644 --- a/skills/anonymizer/SKILL.md +++ b/skills/anonymizer/SKILL.md @@ -48,7 +48,7 @@ regulatory and business context. - **`risk_tolerance` only applies to Rewrite mode**, not Replace. - **`PrivacyGoal.protect` and `.preserve` must each be 10–1000 chars and at least 3 words.** Be specific (categories, named identifiers, structural facets); avoid generic phrasing like "preserve meaning". - **Validator pool is the only model role with built-in load-spreading.** Set `entity_validator: [a, b, c]` in `models.yaml` if rate limits drop rows. Other roles (rewriter, evaluator, etc.) are single-alias. -- **Self-hosted GLiNER:** When detection must not call `build.nvidia.com` (PHI on-prem, air-gapped, latency), use `tools/inference_service.py` from a **source checkout** to compile the pinned `tools/inference_service_profiles/nvidia-gliner.toml` profile and launch its managed plan. The profile uses vLLM 0.26 and the external vLLM Factory project at a pinned source revision. The tool and runtime are not installed by `pip install nemo-anonymizer`; install the repository's `local-models` dependency group first. Add a provider with `endpoint: http://localhost:8001/v1`, then route `entity_detector` through a `gliner-pii-detector` alias with `provider: local-gliner`. Match the compiled host and port in the provider endpoint. `model_configs` is a **complete** model pool, not an overlay. Copy `src/anonymizer/config/default_model_configs/models.yaml` and change only the detector entry, keeping `gpt-oss-120b` and `nemotron-30b-thinking`. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). +- **Self-hosted GLiNER:** When detection must not call `build.nvidia.com` (PHI on-prem, air-gapped, latency), use `tools/inference_service.py` from a **source checkout** to compile the pinned `tools/inference_service_profiles/nvidia-gliner.toml` profile and launch its managed plan. The profile uses vLLM 0.27.1 and the external vLLM Factory project at a pinned source revision. Local serving requires Python 3.12 or later. The tool and runtime are not installed by `pip install nemo-anonymizer`; install the repository's `local-models` dependency group first. Add a provider with `endpoint: http://localhost:8001/v1`, then route `entity_detector` through a `gliner-pii-detector` alias with `provider: local-gliner`. Match the compiled host and port in the provider endpoint. `model_configs` is a **complete** model pool, not an overlay. Copy `src/anonymizer/config/default_model_configs/models.yaml` and change only the detector entry, keeping `gpt-oss-120b` and `nemotron-30b-thinking`. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). - **The evaluation judges use their own model roles** (`entity_coverage_judge`, `detection_validity_judge`, `replace_type_fidelity_judge`, `replace_relational_consistency_judge`, `replace_attribute_fidelity_judge`, `rewrite_judge`), configured in the `evaluate` section of `models.yaml`. They are **not** consumed by `preview()` / `run()`, so a config that anonymizes fine can still fail validation at `evaluate()` if those roles are unset. Defaults ship in `src/anonymizer/config/default_model_configs/evaluate.yaml` (`entity_coverage_judge` defaults to `nemotron-super`). - **Verdict columns are null when the judge was unavailable** — `None` means "unscored", never a pass. `entity_coverage` is a `0–1` float (`1.0` = no missed candidate values or no PII found) or `None`; `missed_entities` lists unique candidate values the anonymizer failed to detect. Replace verdict columns (`type_fidelity_valid`, etc.) are `True` / `False` / `None`. Rewrite `detection_valid` is a `0–1` float fraction (or `None` if unscored). Inspect verdicts per record with `evaluated.display_record(i)`. - **`EvaluateConfig` has one knob today: `compute_detection_validity`** (default `False`). Plain `anonymizer.evaluate(result)` runs entity coverage + the mode's quality judges; pass `EvaluateConfig(compute_detection_validity=True)` only to additionally score detection validity (an internal-facing tag-precision metric). diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index 4a258d38..bc7294f7 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -71,7 +71,7 @@ def build_generation_plan( models.DockerPlacement( host="127.0.0.1", port=8000, - image="vllm/vllm-openai:v0.26.0", + image="vllm/vllm-openai:v0.27.1", gpus="all", ) if docker @@ -154,7 +154,7 @@ def test_compile_vllm_docker_plan_keeps_secrets_symbolic() -> None: placement=models.DockerPlacement( host="127.0.0.1", port=8000, - image="vllm/vllm-openai:v0.26.0", + image="vllm/vllm-openai:v0.27.1", gpus="all", ), access=models.DirectAccess(), @@ -165,7 +165,7 @@ def test_compile_vllm_docker_plan_keeps_secrets_symbolic() -> None: rendered = plan.model_dump_json() assert plan.runtime.kind == "docker" - assert plan.runtime.image == "vllm/vllm-openai:v0.26.0" + assert plan.runtime.image == "vllm/vllm-openai:v0.27.1" assert plan.endpoint.url == "http://127.0.0.1:8000/v1" assert plan.expected_model == "anonymizer-local" assert plan.required_capabilities == ("chat-completions",) @@ -234,7 +234,7 @@ def test_compiler_accepts_gliner_through_external_vllm_factory() -> None: plan = compiler.compile_intent(intent, source_revision="3f68c145") assert plan.declared_capabilities == ("dynamic-labels", "offsets", "scores") - assert "vllm==0.26.0" in plan.dependencies + assert "vllm==0.27.1" in plan.dependencies assert ( "vllm-factory[gliner] @ git+https://github.com/latenceainew/vllm-factory.git@" "7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0" @@ -501,13 +501,22 @@ def test_reference_toml_profiles_are_pinned_and_compile() -> None: assert {path.name for path in profile_paths} == { "gliner2.toml", + "nemotron-3.5-lightning.toml", "nvidia-gliner.toml", "vllm-local.toml", } for path in profile_paths: intent = cli.load_profile(path) assert intent.model.revision is not None - assert compiler.compile_intent(intent, source_revision="3f68c145") + plan = compiler.compile_intent(intent, source_revision="3f68c145") + assert plan + if path.name == "nemotron-3.5-lightning.toml": + assert plan.dependencies == ( + "vllm==0.27.1", + "nvidia-cuda-nvcc==13.0.88", + "nvidia-cuda-crt==13.0.88", + "nvidia-nvvm==13.0.88", + ) def test_probe_records_generation_capabilities() -> None: diff --git a/tests/tools/test_vllm_factory.py b/tests/tools/test_vllm_factory.py index 4414d7ef..8995da27 100644 --- a/tests/tools/test_vllm_factory.py +++ b/tests/tools/test_vllm_factory.py @@ -22,11 +22,15 @@ def test_local_models_group_pins_vllm_and_external_factory_source() -> None: project = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) assert project["dependency-groups"]["local-models"] == [ - "vllm==0.26.0; sys_platform == 'linux'", + "vllm==0.27.1; sys_platform == 'linux' and python_version >= '3.12'", ( "vllm-factory[gliner] @ git+https://github.com/latenceainew/vllm-factory.git@" - "7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0; sys_platform == 'linux'" + "7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0; sys_platform == 'linux' " + "and python_version >= '3.12'" ), + "nvidia-cuda-nvcc==13.0.88; sys_platform == 'linux' and python_version >= '3.12'", + "nvidia-cuda-crt==13.0.88; sys_platform == 'linux' and python_version >= '3.12'", + "nvidia-nvvm==13.0.88; sys_platform == 'linux' and python_version >= '3.12'", ] @@ -62,7 +66,18 @@ def test_parse_server_parameters_accepts_only_the_compiler_contract() -> None: "0.8", "--max-model-len", "2048", + "--max-num-seqs", + "16", "--enforce-eager", + "--enable-prefix-caching", + "--async-scheduling", + "--mamba-backend", + "flashinfer", + "--mamba-ssm-cache-dtype", + "float16", + "--enable-mamba-cache-stochastic-rounding", + "--mamba-cache-philox-rounds", + "5", "--vllm-factory-plugin", "deberta_gliner", ] @@ -72,7 +87,14 @@ def test_parse_server_parameters_accepts_only_the_compiler_contract() -> None: assert parameters.port == 8123 assert parameters.tensor_parallel_size == 2 assert parameters.gpu_memory_utilization == 0.8 + assert parameters.max_num_seqs == 16 assert parameters.enforce_eager is True + assert parameters.enable_prefix_caching is True + assert parameters.async_scheduling is True + assert parameters.mamba_backend == "flashinfer" + assert parameters.mamba_ssm_cache_dtype == "float16" + assert parameters.enable_mamba_cache_stochastic_rounding is True + assert parameters.mamba_cache_philox_rounds == 5 assert parameters.vllm_factory_plugin == "deberta_gliner" @@ -90,7 +112,14 @@ def test_factory_constructs_vllm_frontend_and_async_engine_arguments() -> None: tensor_parallel_size=2, gpu_memory_utilization=0.8, max_model_len=2048, + max_num_seqs=16, enforce_eager=True, + enable_prefix_caching=True, + async_scheduling=True, + mamba_backend="flashinfer", + mamba_ssm_cache_dtype="float16", + enable_mamba_cache_stochastic_rounding=True, + mamba_cache_philox_rounds=5, lora_module="privacy=/models/privacy-adapter", ) @@ -105,7 +134,14 @@ def test_factory_constructs_vllm_frontend_and_async_engine_arguments() -> None: assert arguments.tensor_parallel_size == 2 assert arguments.gpu_memory_utilization == 0.8 assert arguments.max_model_len == 2048 + assert arguments.max_num_seqs == 16 assert arguments.enforce_eager is True + assert arguments.enable_prefix_caching is True + assert arguments.async_scheduling is True + assert arguments.mamba_backend.value == "flashinfer" + assert arguments.mamba_ssm_cache_dtype == "float16" + assert arguments.enable_mamba_cache_stochastic_rounding is True + assert arguments.mamba_cache_philox_rounds == 5 assert arguments.enable_lora is True assert [(module.name, module.path) for module in arguments.lora_modules] == [("privacy", "/models/privacy-adapter")] @@ -132,8 +168,8 @@ def test_factory_constructs_pooling_server_for_external_gliner_plugin() -> None: assert arguments.middleware == [factory.ANONYMIZER_CHAT_MIDDLEWARE] -def test_run_server_uses_the_vllm_0_26_lifecycle_boundary() -> None: - """The process runner imports and invokes vLLM 0.26's relocated setup API.""" +def test_run_server_uses_the_vllm_0_27_lifecycle_boundary() -> None: + """The process runner imports and invokes vLLM 0.27's lifecycle API.""" pytest.importorskip("vllm") uvloop = importlib.import_module("uvloop") api_server = importlib.import_module("vllm.entrypoints.openai.api_server") @@ -181,15 +217,56 @@ def test_factory_avoids_flashinfer_jit_without_overriding_operator_choice() -> N factory = load_factory_module() parameters = factory.VllmServerParameters(model="model", host="127.0.0.1", port=8000) - with mock.patch.dict(os.environ, {}, clear=True): + with ( + mock.patch.object(factory.sys, "version_info", (3, 12)), + mock.patch.dict(os.environ, {}, clear=True), + ): assert factory.prepare_runtime_environment(parameters) == parameters assert os.environ["VLLM_USE_FLASHINFER_SAMPLER"] == "0" - with mock.patch.dict(os.environ, {"VLLM_USE_FLASHINFER_SAMPLER": "1"}, clear=True): + with ( + mock.patch.object(factory.sys, "version_info", (3, 12)), + mock.patch.dict(os.environ, {"VLLM_USE_FLASHINFER_SAMPLER": "1"}, clear=True), + ): assert factory.prepare_runtime_environment(parameters) == parameters assert os.environ["VLLM_USE_FLASHINFER_SAMPLER"] == "1" +def test_factory_rejects_python_3_11_before_importing_vllm() -> None: + """The server reports the local vLLM Python floor before vLLM starts.""" + factory = load_factory_module() + parameters = factory.VllmServerParameters(model="model", host="127.0.0.1", port=8000) + + with mock.patch.object(factory.sys, "version_info", (3, 11)): + with pytest.raises(RuntimeError, match="Python 3.12 or later"): + factory.prepare_runtime_environment(parameters) + + +def test_factory_configures_packaged_cuda_for_flashinfer(tmp_path: Path) -> None: + """The FlashInfer backend can JIT from the CUDA toolkit shipped as Python wheels.""" + factory = load_factory_module() + cuda_root = tmp_path / "nvidia" / "cu13" + (cuda_root / "bin").mkdir(parents=True) + (cuda_root / "bin" / "nvcc").touch() + (cuda_root / "lib").mkdir() + runtime = cuda_root / "lib" / "libcudart.so.13" + runtime.touch() + + with ( + mock.patch.object(factory, "_packaged_cuda_root", return_value=cuda_root), + mock.patch.object(factory.Path, "home", return_value=tmp_path), + mock.patch.dict(os.environ, {}, clear=True), + ): + factory.configure_flashinfer_toolchain() + + root_digest = factory.hashlib.sha256(os.fsencode(cuda_root.resolve())).hexdigest()[:16] + link_directory = tmp_path / ".cache" / "nemo-anonymizer" / "cuda-link" / root_digest + assert os.environ["CUDA_HOME"] == str(cuda_root) + assert os.environ["LIBRARY_PATH"] == str(link_directory) + assert os.environ["LD_LIBRARY_PATH"] == str(cuda_root / "lib") + assert (link_directory / "libcudart.so").resolve() == runtime + + def test_factory_selects_model_and_io_plugins_together() -> None: """vLLM's shared plugin allowlist retains both factory entry-point groups.""" factory = load_factory_module() @@ -203,6 +280,7 @@ def test_factory_selects_model_and_io_plugins_together() -> None: with ( mock.patch.object(factory, "prepare_model", return_value="/tmp/prepared"), + mock.patch.object(factory.sys, "version_info", (3, 12)), mock.patch.dict(os.environ, {}, clear=True), ): factory.prepare_runtime_environment(parameters) diff --git a/tools/inference_service_compiler/compiler.py b/tools/inference_service_compiler/compiler.py index f29525a2..6c9e1bef 100644 --- a/tools/inference_service_compiler/compiler.py +++ b/tools/inference_service_compiler/compiler.py @@ -39,6 +39,12 @@ ) VLLM_API_KEY_ENV = "VLLM_API_KEY" +VLLM_DEPENDENCY = "vllm==0.27.1" +FLASHINFER_CUDA_TOOLCHAIN_DEPENDENCIES = ( + "nvidia-cuda-nvcc==13.0.88", + "nvidia-cuda-crt==13.0.88", + "nvidia-nvvm==13.0.88", +) class CompilerDiagnostic(FrozenModel): @@ -353,8 +359,22 @@ def _vllm_engine_arguments(intent: InferenceIntent, engine: VllmEngine) -> tuple arguments.extend(_literal_arguments("--gpu-memory-utilization", str(engine.gpu_memory_utilization))) if engine.max_model_len is not None: arguments.extend(_literal_arguments("--max-model-len", str(engine.max_model_len))) + if engine.max_num_seqs is not None: + arguments.extend(_literal_arguments("--max-num-seqs", str(engine.max_num_seqs))) if engine.eager: arguments.extend(_literal_arguments("--enforce-eager")) + if engine.enable_prefix_caching: + arguments.extend(_literal_arguments("--enable-prefix-caching")) + if engine.async_scheduling: + arguments.extend(_literal_arguments("--async-scheduling")) + if engine.mamba_backend is not None: + arguments.extend(_literal_arguments("--mamba-backend", engine.mamba_backend)) + if engine.mamba_ssm_cache_dtype != "auto": + arguments.extend(_literal_arguments("--mamba-ssm-cache-dtype", engine.mamba_ssm_cache_dtype)) + if engine.enable_mamba_cache_stochastic_rounding: + arguments.extend(_literal_arguments("--enable-mamba-cache-stochastic-rounding")) + if engine.mamba_cache_philox_rounds: + arguments.extend(_literal_arguments("--mamba-cache-philox-rounds", str(engine.mamba_cache_philox_rounds))) if engine.factory is not None: arguments.extend( _literal_arguments( @@ -392,9 +412,11 @@ def _literal_arguments(*values: str) -> tuple[CommandArgument, ...]: def _plan_dependencies(intent: InferenceIntent) -> tuple[str, ...]: if isinstance(intent.engine, VllmEngine): - dependencies = ["vllm==0.26.0"] + dependencies = [VLLM_DEPENDENCY] if intent.engine.factory is not None: dependencies.append(VLLM_FACTORY_DEPENDENCY) + if intent.engine.mamba_backend == "flashinfer": + dependencies.extend(FLASHINFER_CUDA_TOOLCHAIN_DEPENDENCIES) return tuple(dependencies) return () diff --git a/tools/inference_service_compiler/models.py b/tools/inference_service_compiler/models.py index 6135337f..c855a07f 100644 --- a/tools/inference_service_compiler/models.py +++ b/tools/inference_service_compiler/models.py @@ -107,7 +107,14 @@ class VllmEngine(FrozenModel): tensor_parallel_size: int | None = Field(default=None, ge=1) gpu_memory_utilization: float | None = Field(default=None, gt=0, le=1) max_model_len: int | None = Field(default=None, ge=1) + max_num_seqs: int | None = Field(default=None, ge=1) eager: bool = False + enable_prefix_caching: bool = False + async_scheduling: bool = False + mamba_backend: Literal["triton", "flashinfer"] | None = None + mamba_ssm_cache_dtype: Literal["auto", "float32", "float16", "bfloat16"] = "auto" + enable_mamba_cache_stochastic_rounding: bool = False + mamba_cache_philox_rounds: int = Field(default=0, ge=0) factory: VllmFactoryIntegration | None = None diff --git a/tools/inference_service_compiler/vllm_runtime.py b/tools/inference_service_compiler/vllm_runtime.py index 7924c380..1b73107f 100644 --- a/tools/inference_service_compiler/vllm_runtime.py +++ b/tools/inference_service_compiler/vllm_runtime.py @@ -5,13 +5,16 @@ from __future__ import annotations import argparse +import hashlib import importlib +import importlib.util import os import sys from argparse import Namespace from collections.abc import Sequence from dataclasses import dataclass, replace from pathlib import Path +from typing import Literal from inference_service_compiler.vllm_factory_integration import ( io_processor_for, @@ -19,6 +22,7 @@ ) ANONYMIZER_CHAT_MIDDLEWARE = "inference_service_compiler.vllm_factory_adapter.anonymizer_chat_compatibility" +MINIMUM_VLLM_PYTHON = (3, 12) @dataclass(frozen=True) @@ -34,7 +38,14 @@ class VllmServerParameters: tensor_parallel_size: int | None = None gpu_memory_utilization: float | None = None max_model_len: int | None = None + max_num_seqs: int | None = None enforce_eager: bool = False + enable_prefix_caching: bool = False + async_scheduling: bool = False + mamba_backend: Literal["triton", "flashinfer"] | None = None + mamba_ssm_cache_dtype: Literal["auto", "float32", "float16", "bfloat16"] = "auto" + enable_mamba_cache_stochastic_rounding: bool = False + mamba_cache_philox_rounds: int = 0 lora_module: str | None = None vllm_factory_plugin: str | None = None prepared_model_root: str = "/tmp/anonymizer-vllm-factory" @@ -52,7 +63,18 @@ def parse_server_parameters(argv: Sequence[str]) -> VllmServerParameters: parser.add_argument("--tensor-parallel-size", type=int) parser.add_argument("--gpu-memory-utilization", type=float) parser.add_argument("--max-model-len", type=int) + parser.add_argument("--max-num-seqs", type=int) parser.add_argument("--enforce-eager", action="store_true") + parser.add_argument("--enable-prefix-caching", action="store_true") + parser.add_argument("--async-scheduling", action="store_true") + parser.add_argument("--mamba-backend", choices=("triton", "flashinfer")) + parser.add_argument( + "--mamba-ssm-cache-dtype", + choices=("auto", "float32", "float16", "bfloat16"), + default="auto", + ) + parser.add_argument("--enable-mamba-cache-stochastic-rounding", action="store_true") + parser.add_argument("--mamba-cache-philox-rounds", type=int, default=0) parser.add_argument("--enable-lora", action="store_true") parser.add_argument("--lora-modules") parser.add_argument( @@ -73,7 +95,14 @@ def parse_server_parameters(argv: Sequence[str]) -> VllmServerParameters: tensor_parallel_size=parsed.tensor_parallel_size, gpu_memory_utilization=parsed.gpu_memory_utilization, max_model_len=parsed.max_model_len, + max_num_seqs=parsed.max_num_seqs, enforce_eager=parsed.enforce_eager, + enable_prefix_caching=parsed.enable_prefix_caching, + async_scheduling=parsed.async_scheduling, + mamba_backend=parsed.mamba_backend, + mamba_ssm_cache_dtype=parsed.mamba_ssm_cache_dtype, + enable_mamba_cache_stochastic_rounding=parsed.enable_mamba_cache_stochastic_rounding, + mamba_cache_philox_rounds=parsed.mamba_cache_philox_rounds, lora_module=parsed.lora_modules, vllm_factory_plugin=parsed.vllm_factory_plugin, prepared_model_root=parsed.prepared_model_root, @@ -83,6 +112,7 @@ def parse_server_parameters(argv: Sequence[str]) -> VllmServerParameters: def build_server_arguments(parameters: VllmServerParameters) -> Namespace: """Construct vLLM frontend and async-engine configs through its Python API.""" arg_utils = importlib.import_module("vllm.engine.arg_utils") + mamba_config = importlib.import_module("vllm.config.mamba") cli_args = importlib.import_module("vllm.entrypoints.openai.cli_args") model_protocol = importlib.import_module("vllm.entrypoints.openai.models.protocol") @@ -101,14 +131,20 @@ def build_server_arguments(parameters: VllmServerParameters) -> Namespace: served_model_name=[parameters.served_model_name] if parameters.served_model_name is not None else None, tensor_parallel_size=parameters.tensor_parallel_size or 1, gpu_memory_utilization=parameters.gpu_memory_utilization or 0.9, + max_num_seqs=parameters.max_num_seqs, enforce_eager=parameters.enforce_eager, enable_lora=lora_modules is not None, runner="pooling" if factory_plugin is not None else "auto", trust_remote_code=factory_plugin is not None, dtype="bfloat16" if factory_plugin is not None else "auto", - enable_prefix_caching=False if factory_plugin is not None else None, + enable_prefix_caching=False if factory_plugin is not None else parameters.enable_prefix_caching, enable_chunked_prefill=False if factory_plugin is not None else None, io_processor_plugin=io_processor_for(factory_plugin) if factory_plugin is not None else None, + async_scheduling=parameters.async_scheduling, + mamba_backend=mamba_config.MambaBackendEnum[(parameters.mamba_backend or "triton").upper()], + mamba_ssm_cache_dtype=parameters.mamba_ssm_cache_dtype, + enable_mamba_cache_stochastic_rounding=parameters.enable_mamba_cache_stochastic_rounding, + mamba_cache_philox_rounds=parameters.mamba_cache_philox_rounds, ) if parameters.max_model_len is not None: engine.max_model_len = parameters.max_model_len @@ -139,10 +175,74 @@ def expose_interpreter_tools() -> None: os.environ["PATH"] = os.pathsep.join([interpreter_bin, *path_entries]) +def _prepend_environment_path(name: str, path: Path) -> None: + current = os.environ.get(name, "") + entries = [entry for entry in current.split(os.pathsep) if entry and entry != str(path)] + os.environ[name] = os.pathsep.join([str(path), *entries]) + + +def _packaged_cuda_root() -> Path | None: + """Find the CUDA toolkit installed beside the selected Python interpreter.""" + package = importlib.util.find_spec("nvidia") + if package is None or package.submodule_search_locations is None: + return None + for location in package.submodule_search_locations: + candidate = Path(location) / "cu13" + if (candidate / "bin" / "nvcc").is_file() and (candidate / "lib").is_dir(): + return candidate + return None + + +def _cudart_link_directory(cuda_root: Path) -> Path: + """Expose the versioned CUDA wheel runtime under the linker name FlashInfer expects.""" + library_directory = cuda_root / "lib" + unversioned = library_directory / "libcudart.so" + if unversioned.exists(): + return library_directory + candidates = sorted(library_directory.glob("libcudart.so.*")) + if not candidates: + raise RuntimeError(f"CUDA toolkit at {cuda_root} does not contain libcudart") + root_digest = hashlib.sha256(os.fsencode(cuda_root.resolve())).hexdigest()[:16] + link_directory = Path.home() / ".cache" / "nemo-anonymizer" / "cuda-link" / root_digest + link_directory.mkdir(parents=True, exist_ok=True) + link = link_directory / "libcudart.so" + target = candidates[-1].resolve() + if link.is_symlink() and link.resolve() == target: + return link_directory + if link.exists() or link.is_symlink(): + link.unlink() + try: + link.symlink_to(target) + except FileExistsError: + if not link.is_symlink() or link.resolve() != target: + raise + return link_directory + + +def configure_flashinfer_toolchain() -> None: + """Configure FlashInfer JIT from the CUDA wheels pinned with vLLM.""" + configured = os.environ.get("CUDA_HOME") + if configured: + compiler = Path(configured) / "bin" / "nvcc" + if not compiler.is_file(): + raise RuntimeError(f"CUDA_HOME does not contain bin/nvcc: {configured}") + return + cuda_root = _packaged_cuda_root() + if cuda_root is None: + raise RuntimeError("FlashInfer Mamba requires a CUDA compiler; install the local-models group or set CUDA_HOME") + os.environ["CUDA_HOME"] = str(cuda_root) + _prepend_environment_path("LIBRARY_PATH", _cudart_link_directory(cuda_root)) + _prepend_environment_path("LD_LIBRARY_PATH", cuda_root / "lib") + + def prepare_runtime_environment(parameters: VllmServerParameters) -> VllmServerParameters: """Prepare the selected stock vLLM or vLLM Factory runtime.""" + if sys.version_info < MINIMUM_VLLM_PYTHON: + raise RuntimeError("vLLM 0.27.1 local serving requires Python 3.12 or later") expose_interpreter_tools() os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0") + if parameters.mamba_backend == "flashinfer": + configure_flashinfer_toolchain() plugin = parameters.vllm_factory_plugin if plugin is None: return parameters diff --git a/tools/inference_service_profiles/nemotron-3.5-lightning.toml b/tools/inference_service_profiles/nemotron-3.5-lightning.toml new file mode 100644 index 00000000..e16423b6 --- /dev/null +++ b/tools/inference_service_profiles/nemotron-3.5-lightning.toml @@ -0,0 +1,37 @@ +schema_version = "inference-service.intent/v1" + +[task] +kind = "generation" +chat = true + +[model] +kind = "hugging-face" +model_id = "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16" +revision = "33268dc8a6da85a56be2b12241453e4e1237bbe1" + +[engine] +kind = "vllm" +python_executable = ".venv/bin/python" +served_model_name = "nemotron-3.5-lightning-local" +gpu_memory_utilization = 0.88 +max_model_len = 8192 +max_num_seqs = 16 +enable_prefix_caching = true +async_scheduling = true +mamba_backend = "flashinfer" +mamba_ssm_cache_dtype = "float16" +enable_mamba_cache_stochastic_rounding = true +mamba_cache_philox_rounds = 5 + +[placement] +kind = "local-process" +host = "127.0.0.1" +port = 8000 + +[access] +kind = "direct" + +[lifecycle] +kind = "managed" +startup_timeout_seconds = 1800 +shutdown_timeout_seconds = 60 diff --git a/uv.lock b/uv.lock index 3d6e1ec6..3bc6d52d 100644 --- a/uv.lock +++ b/uv.lock @@ -205,25 +205,25 @@ wheels = [ [[package]] name = "apache-tvm-ffi" -version = "0.1.10" +version = "0.1.11" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/b0/5114e30faffe3279a51a5f3b45dd1b7ce09af1246b62447b45a39a374e54/apache_tvm_ffi-0.1.10.tar.gz", hash = "sha256:974c208766c304c780c17c6d405449e862f83b22c7b6b2b8c28b29d55a806ae3", size = 2691605, upload-time = "2026-04-07T19:58:51.767Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/3d/4b9226cd45aa800a6904603dda9b323d728f3c3869952a673f3483b78b19/apache_tvm_ffi-0.1.11.tar.gz", hash = "sha256:153cd2c5a9717804cb0bcd9b2709f22a1e5f80ed05b5a490faf5949b136eedba", size = 2798354, upload-time = "2026-05-04T17:48:43.852Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/55/c3/598da8bf49e850aa329a024929643eb141d7907f4d97705b74e49ca499f6/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d5cf055a83e1b1944dd05386c593bc22de29a1aeb6cae45af54735796875194a", size = 2543849, upload-time = "2026-04-07T19:58:05.419Z" }, - { url = "https://files.pythonhosted.org/packages/50/58/221b41c5f77405f99875754f2a38c01da49387e366bf0fd40302b2cd25f3/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:81c4144fc06750312f2829960862bd52ba6f0bb17e6d7aae3f7a09f9170f7e7a", size = 2650260, upload-time = "2026-04-07T19:58:07.002Z" }, - { url = "https://files.pythonhosted.org/packages/01/2b/36b5210d24492dc4dda488d785dd4039c0788238f6aa4aa5067b2ea494d1/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7bafe9a6191c77f3978e9cd9726799abbe7fd574913fa2416402bc876633524e", size = 2459987, upload-time = "2026-04-07T19:58:08.409Z" }, - { url = "https://files.pythonhosted.org/packages/9f/36/8f8f719c1c52ed978fc99acde51827f5fc48380e69a310a02a6a5ae94d0f/apache_tvm_ffi-0.1.10-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a2ba653825f806a87fe2ca48ebab1abb9ae0f17d6642fbada622c6c5eea9fe96", size = 2631364, upload-time = "2026-04-07T19:58:09.784Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2a/1978a1c827e1212de4f369ec08cfeb44719bbe6cbeab90b15e967c68c108/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ec5c4a81e294e6379e4dea68c86266924d3f22829c3de272806c980238e43e59", size = 2476596, upload-time = "2026-04-07T19:58:14.316Z" }, - { url = "https://files.pythonhosted.org/packages/50/6f/23740f06829030704e6f8f1f7093a06b7a68f904baa40053a5f594705bae/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:73d478395a8625dd92fde7b7fd92b4719f18f480b78336e422cb66cc7985213d", size = 2589574, upload-time = "2026-04-07T19:58:15.94Z" }, - { url = "https://files.pythonhosted.org/packages/92/d0/54badf5c8f6208e06f331a20ddd154f19c94c2e906da5b8cce7d60727d4b/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3829216a8500c2f61062e48c627f6db6c3fa49416b3ffa85bc04243ae5d759f7", size = 2396434, upload-time = "2026-04-07T19:58:17.519Z" }, - { url = "https://files.pythonhosted.org/packages/51/f7/ca3fdadc2468e8b67a2f3f13bb7aa132c584feefd8a25dbf920e4bf0a03b/apache_tvm_ffi-0.1.10-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:96b69030c722572e13e30182733adfa2d604258e988b3f6630a16f397c7f9288", size = 2571084, upload-time = "2026-04-07T19:58:20.399Z" }, - { url = "https://files.pythonhosted.org/packages/2e/5d/b1661512164772fc9ef1642234bf117182b440fc0a0b2ca8bd829fe7b40e/apache_tvm_ffi-0.1.10-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32b9f4a44c09fcdd0994ee3c4415bf0371d68ea35a46da94ddcc666c9a6cf677", size = 2508518, upload-time = "2026-04-07T19:58:25.3Z" }, - { url = "https://files.pythonhosted.org/packages/d2/57/7266807b34344b9d8e4d776ebff38fd25f93a73e8c24bc595a67b6b69b3c/apache_tvm_ffi-0.1.10-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c9b93dc7fdc99d4cc44e9ac95063073b4fb8ced94929197ea3d631b70f554d8a", size = 2617108, upload-time = "2026-04-07T19:58:26.888Z" }, - { url = "https://files.pythonhosted.org/packages/96/c3/a152ed68f57a491baaf70819224b98643309c7488fdcbc6fa3c84ebb9ca8/apache_tvm_ffi-0.1.10-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:74724db54dfb825951e2deb3d2024b2c1867bff456db81512e475f9ccdd9b86b", size = 2432434, upload-time = "2026-04-07T19:58:28.681Z" }, - { url = "https://files.pythonhosted.org/packages/c4/09/5e2877c635edc8ac83caa106a6e78bd4816cbc2e52e1daea652c1fe956cf/apache_tvm_ffi-0.1.10-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ac03c04145d9c248992e6f2ec2392a6914966a416eeeeaa729393f40b047be42", size = 2602517, upload-time = "2026-04-07T19:58:30.35Z" }, + { url = "https://files.pythonhosted.org/packages/8f/22/aec1d70baa4bc1e3962a23439f82099f7775992cc5b70a19d4a8ef2a47e4/apache_tvm_ffi-0.1.11-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2f0d4b165f371d2dee6013e47353d178b01742171fd1092c654cbbc0fa5c6d60", size = 2675459, upload-time = "2026-05-04T17:47:48.63Z" }, + { url = "https://files.pythonhosted.org/packages/9c/b6/84acc663a43ba6e72b4dfd8d923fb5a2d1eb2c867b5b2067c18cb3dc855a/apache_tvm_ffi-0.1.11-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2cf501753d7693daa73711a27f0f9d9f0f76e9e7d98f2fc2403f423ee7bbfd9b", size = 2789376, upload-time = "2026-05-04T17:47:50.449Z" }, + { url = "https://files.pythonhosted.org/packages/ac/96/e216d5d0f420ccf54775c913b6506755f65bc262511a9948b4d1387bcbc9/apache_tvm_ffi-0.1.11-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a051c84985be3f9d8a20a16ec4bdba73a7ae01d3fb2f18a2c72bbd7a28aaa155", size = 2584233, upload-time = "2026-05-04T17:47:52.387Z" }, + { url = "https://files.pythonhosted.org/packages/72/9c/af12a5e796a672664f2f18eca989222697b91974a9c9e98d4ec2ecfeeb83/apache_tvm_ffi-0.1.11-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:87f84e7c2393fadac340fd179a631a697effe54d7317b2543e0930452a0a673d", size = 2768221, upload-time = "2026-05-04T17:47:54.057Z" }, + { url = "https://files.pythonhosted.org/packages/2a/a9/f48e5dd4ae1f6f0c5ffac259c0a9531b7d6a7c0a4c45bc2229d55de6adf8/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:da2c8d07fdc737d1ba75f4de25c29f156905b9dc980f1da90c395b4db525f522", size = 2605176, upload-time = "2026-05-04T17:47:59.676Z" }, + { url = "https://files.pythonhosted.org/packages/36/99/2848df4e8ed5bf51df1d286d1718510584fa61e88adbc9c5b23d71b38f7c/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:78aa1857b04a2ea718317041ab3f01288b3d496e6036eb1b99ebdc9da0fdaef5", size = 2725887, upload-time = "2026-05-04T17:48:01.381Z" }, + { url = "https://files.pythonhosted.org/packages/7d/80/963c991934a4eb0fa0c0178f51963333fe14a96b732009da642b6bf6b42e/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a8b845c8dff498fb981c1dda36c954549204191b485a385845e604966594d0b2", size = 2513121, upload-time = "2026-05-04T17:48:03.43Z" }, + { url = "https://files.pythonhosted.org/packages/4d/18/95569107ee83619d61a3bb0d28743a0599f85c5161981e3e098c82c2b185/apache_tvm_ffi-0.1.11-cp312-abi3-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2843f084cdc94dedacd8b257a395a2b71b8a3dc7fc99711b148bf1d161983128", size = 2697683, upload-time = "2026-05-04T17:48:05.222Z" }, + { url = "https://files.pythonhosted.org/packages/9d/d1/dc0c26cf68635a1184ba39cccb6cb3cf9675c7030f135f47205e56bdd2b6/apache_tvm_ffi-0.1.11-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a05b36530d7cd5bb93b1a21a3b81ff060968c20456c4870b1a80d65966d5114f", size = 2639857, upload-time = "2026-05-04T17:48:11.1Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ad/4e3d4c5ec36e2ecadf6e5eb81cde065c69218cf722606b73af0ea6fdab75/apache_tvm_ffi-0.1.11-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9b158f93bdfc497ead9fce5ffdd4d132708de60970ffc97d890dd62fa39d9fb4", size = 2755683, upload-time = "2026-05-04T17:48:13.016Z" }, + { url = "https://files.pythonhosted.org/packages/51/37/54deceea6bac0e93844bd572a2fae8549e86e6309c732a0acaeb07a88c6b/apache_tvm_ffi-0.1.11-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0f77406e2773ad18109369417b5ccf6aee3c813867dbd5d2d97170bfa7b491f1", size = 2552014, upload-time = "2026-05-04T17:48:14.692Z" }, + { url = "https://files.pythonhosted.org/packages/14/e8/52c9544be5850c7c0e5edce08f2dc9d05c3ecb10b7ae9b3a9313d1b2857e/apache_tvm_ffi-0.1.11-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78f0c9dc69727665de58faebacf6a3f4a1d75a355591e963e1bc691fc9bf5cd5", size = 2730358, upload-time = "2026-05-04T17:48:16.39Z" }, ] [[package]] @@ -379,9 +379,6 @@ wheels = [ name = "blake3" version = "1.0.9" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.12'" }, -] sdist = { url = "https://files.pythonhosted.org/packages/26/6a/4cc5a9dd40fd8a6d283fd3761e5f59c490109571ef8e3c73245417e5a305/blake3-1.0.9.tar.gz", hash = "sha256:5fa374fa5070ca084368776c19b420157eb0f2d3f091343d6bc59189929d62e2", size = 116872, upload-time = "2026-06-22T18:02:25.366Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b9/d6/d5462ec19a7f3d084fe327e08618fa107799ee708df04b3a2d620bd62816/blake3-1.0.9-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ee96daaa850700fd342a811fa10a8780fd2e8464a71b83a1779c7b6becd3dd5", size = 377621, upload-time = "2026-06-22T18:00:18.389Z" }, @@ -954,45 +951,51 @@ wheels = [ [[package]] name = "cuda-toolkit" -version = "13.0.2" +version = "13.0.3.0" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, + { url = "https://files.pythonhosted.org/packages/d1/c7/a79086a62c98befcdb8349656c6f114e2db3b8b2422f6e25c97a7f2a9a3c/cuda_toolkit-13.0.3.0-py2.py3-none-any.whl", hash = "sha256:d693caaa261214ddd7dbb60d68e71cbed884e68c2be7509778f3051da0b91c3f", size = 2512, upload-time = "2026-04-14T00:50:08.173Z" }, ] [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cudart = [ - { name = "nvidia-cuda-runtime" }, + { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cufft = [ - { name = "nvidia-cufft" }, + { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cufile = [ - { name = "nvidia-cufile" }, + { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cupti = [ - { name = "nvidia-cuda-cupti" }, + { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] curand = [ - { name = "nvidia-curand" }, + { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cusolver = [ - { name = "nvidia-cusolver" }, + { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] cusparse = [ - { name = "nvidia-cusparse" }, + { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] nvtx = [ - { name = "nvidia-nvtx" }, + { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, ] [[package]] @@ -1526,13 +1529,15 @@ wheels = [ [[package]] name = "flashinfer-python" -version = "0.6.14" +version = "0.6.16.post3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-tvm-ffi" }, { name = "click" }, + { name = "cuda-python" }, { name = "cuda-tile" }, { name = "einops" }, + { name = "nccl4py" }, { name = "ninja" }, { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, @@ -1545,9 +1550,9 @@ dependencies = [ { name = "torch" }, { name = "tqdm" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8f/11/ce2271271bee6990d34ed2d01288e9e92a0ea8ee45fb28de8e746c7da761/flashinfer_python-0.6.14.tar.gz", hash = "sha256:f4da8b5e005601784e85e0dcaa3389f908ee2d32c2560142d67124ab10e4a070", size = 9944949, upload-time = "2026-07-02T00:22:50.879Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c2/46/17045cb47b7b93fcb4e5303e398e6b129b86239930ac0875942a83d8b962/flashinfer_python-0.6.16.post3.tar.gz", hash = "sha256:9b146cfcc1454c80f3f99444db48013b3eadaeb32f2a399bd76c9471ecb72f04", size = 11076656, upload-time = "2026-08-08T01:14:16.408Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/8f/b101913cb2b3687654f56681cfe9836d447526be663c149966470ef70531/flashinfer_python-0.6.14-py3-none-any.whl", hash = "sha256:d124369346a3d48eac67e31c42f7a3c813bcc0abc10e2e36db413b7b3dfd97df", size = 14574383, upload-time = "2026-07-02T00:22:48.413Z" }, + { url = "https://files.pythonhosted.org/packages/9d/dc/5367cb601fc9190cc75b4c141f5f7e9a224d1c26a1e983151823e02a25b3/flashinfer_python-0.6.16.post3-py3-none-any.whl", hash = "sha256:caf686b9b079abe1c9d65ab505698bd325e8072de40afd822f2c74f2ac3bc601", size = 15836034, upload-time = "2026-08-08T01:14:13.709Z" }, ] [[package]] @@ -3437,6 +3442,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a9/82/0340caa499416c78e5d8f5f05947ae4bc3cba53c9f038ab6e9ed964e22f1/nbformat-5.10.4-py3-none-any.whl", hash = "sha256:3b48d6c8fbca4b299bf3982ea7db1af21580e4fec269ad087b9e81588891200b", size = 78454, upload-time = "2024-04-04T11:20:34.895Z" }, ] +[[package]] +name = "nccl4py" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-core" }, + { name = "cuda-pathfinder" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "numpy", version = "2.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + { name = "packaging" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/79/c5779d2d89e398ba480e34e7291c4f767a810ba599384bce10e2d46cd352/nccl4py-0.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8e247d3a2b6e0253567322d518ec94320caa92649475416abceefaf0bc2db71", size = 10874554, upload-time = "2026-06-11T20:38:29.541Z" }, + { url = "https://files.pythonhosted.org/packages/13/70/9761ba36d32e6e3a54f090fae29f47092d2ea5ff682bfec75201e051fee4/nccl4py-0.3.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7945acf8c0224f727f87db40eb3a6060ad7809256b0f37dde6b94d9d8fa6b65c", size = 10972178, upload-time = "2026-06-11T20:38:29.825Z" }, + { url = "https://files.pythonhosted.org/packages/a9/8e/ef2b349050c2586d9c78fb2038b05f694bacae7e61c3cf0805a662f82ae7/nccl4py-0.3.1-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e0b1bab08b374ba21bb36612710866173e703c06dc197ab13b1e093436ac27ce", size = 10898883, upload-time = "2026-06-11T20:38:34.091Z" }, + { url = "https://files.pythonhosted.org/packages/06/72/b956d2d6991d4c152442a56a072eb88c09b6fa6b46fd449ed3f861ff55cc/nccl4py-0.3.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f96117a0aed13744d2636760962f1cb45be9138846023b69c5a8053e531cc76", size = 11056754, upload-time = "2026-06-11T20:38:35.921Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e7/f8048544ccca9a1ff72b5c602529a67868059d8a65d961197780e30d6c24/nccl4py-0.3.1-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50068bb4e6f60dd831b2d394a9789f08a6f4346f0e51a9d717536868a8fdd398", size = 10836501, upload-time = "2026-06-11T20:38:33.237Z" }, + { url = "https://files.pythonhosted.org/packages/20/6b/84a2eb82666136fcbe93461029fec727ddf61cf2b54687190450c58c85c7/nccl4py-0.3.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7fd7862777777162f4735b85472951d91b8847cd7010d8f60bfefe58c7285583", size = 10999870, upload-time = "2026-06-11T20:38:33.78Z" }, + { url = "https://files.pythonhosted.org/packages/91/5f/992ff72ead3c11ba4abc35c34ea67a6eaf6d6fca922d2d992319c7b4666f/nccl4py-0.3.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13db9f786c7919ed1df7079c03acbe49eb569d5625fd5cec2344075f37b5e140", size = 10828564, upload-time = "2026-06-11T20:38:34.619Z" }, + { url = "https://files.pythonhosted.org/packages/49/03/bf540fddca3c803ff520d3e984e413c2b0fdba824f961771251f6cbf977a/nccl4py-0.3.1-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5b684a3ac083fd76bf1e57e69e76575502bcd2aface23e2dad7a88461ce8916d", size = 10957154, upload-time = "2026-06-11T20:38:40.366Z" }, + { url = "https://files.pythonhosted.org/packages/ba/07/96b85a386a6643766f6755f4951aecb603d1d391527f1bb8db3e849f71df/nccl4py-0.3.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa09f12a93e0eb7b2dbbffbcb0a3daeeb3c211f2433cc6cc896faeec85ea4b2e", size = 10924309, upload-time = "2026-06-11T20:38:41.897Z" }, + { url = "https://files.pythonhosted.org/packages/1d/43/1dbdadddb88e53c08875c7ede540f95ebb96d81bc8661281179a2d033d28/nccl4py-0.3.1-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c58bf4db2eb27636587b3ce899fe47d72c16bf1eeb76c5e1c9bb1feaf57048f5", size = 10902042, upload-time = "2026-06-11T20:38:42.887Z" }, +] + [[package]] name = "nemo-anonymizer" source = { editable = "." } @@ -3471,8 +3501,11 @@ docs = [ { name = "mkdocstrings", extra = ["python"] }, ] local-models = [ - { name = "vllm", marker = "sys_platform == 'linux'" }, - { name = "vllm-factory", extra = ["gliner"], marker = "sys_platform == 'linux'" }, + { name = "nvidia-cuda-crt", marker = "python_full_version >= '3.12' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvcc", marker = "python_full_version >= '3.12' and sys_platform == 'linux'" }, + { name = "nvidia-nvvm", marker = "python_full_version >= '3.12' and sys_platform == 'linux'" }, + { name = "vllm", marker = "python_full_version >= '3.12' and sys_platform == 'linux'" }, + { name = "vllm-factory", extra = ["gliner"], marker = "python_full_version >= '3.12' and sys_platform == 'linux'" }, ] measurement = [ { name = "wandb", extra = ["workspaces"] }, @@ -3517,8 +3550,11 @@ docs = [ { name = "mkdocstrings", extras = ["python"] }, ] local-models = [ - { name = "vllm", marker = "sys_platform == 'linux'", specifier = "==0.26.0" }, - { name = "vllm-factory", extras = ["gliner"], marker = "sys_platform == 'linux'", git = "https://github.com/latenceainew/vllm-factory.git?rev=7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0" }, + { name = "nvidia-cuda-crt", marker = "python_full_version >= '3.12' and sys_platform == 'linux'", specifier = "==13.0.88" }, + { name = "nvidia-cuda-nvcc", marker = "python_full_version >= '3.12' and sys_platform == 'linux'", specifier = "==13.0.88" }, + { name = "nvidia-nvvm", marker = "python_full_version >= '3.12' and sys_platform == 'linux'", specifier = "==13.0.88" }, + { name = "vllm", marker = "python_full_version >= '3.12' and sys_platform == 'linux'", specifier = "==0.27.1" }, + { name = "vllm-factory", extras = ["gliner"], marker = "python_full_version >= '3.12' and sys_platform == 'linux'", git = "https://github.com/latenceainew/vllm-factory.git?rev=7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0" }, ] measurement = [{ name = "wandb", extras = ["workspaces"], specifier = ">=0.19,<1" }] notebooks = [ @@ -3803,11 +3839,14 @@ wheels = [ [[package]] name = "nvidia-cublas" -version = "13.1.0.3" +version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cuda-nvrtc" }, +] wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/a5/fce49e2ae977e0ccc084e5adafceb4f0ac0c8333cb6863501618a7277f67/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c86fc7f7ae36d7528288c5d88098edcb7b02c633d262e7ddbb86b0ad91be5df2", size = 542851226, upload-time = "2025-10-09T08:59:04.818Z" }, - { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236, upload-time = "2025-10-09T08:59:32.536Z" }, + { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/154ca20c38269e05eff77c1464e6c1da89f50a6390b565e9d82e06bc11e1/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:37936a16db8fe4ac1f065c2139360608a543a09275cb1a1af612e08cfa065436", size = 423138758, upload-time = "2026-04-08T18:46:58.655Z" }, ] [[package]] @@ -3821,11 +3860,11 @@ wheels = [ [[package]] name = "nvidia-cuda-crt" -version = "13.3.73" +version = "13.0.88" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/41/2089e411507d66458d67208bdd1bc562d492bb6458c3d2aea4603072a219/nvidia_cuda_crt-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:60aacc0b5e1e8b40c62abe4d1ab16440add91b99bd2f17f62dd091586b73d166", size = 157353, upload-time = "2026-06-29T16:42:38.163Z" }, - { url = "https://files.pythonhosted.org/packages/7e/ce/16d76f4b5b3f7460f5ebd17516685495c149c66651ffdd381f90e4d4e65c/nvidia_cuda_crt-13.3.73-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:df14a17ae1c5c3171265411212246654d780f89344ea85344466c6b955247543", size = 157352, upload-time = "2026-06-29T16:43:09.209Z" }, + { url = "https://files.pythonhosted.org/packages/49/b9/2cb230193e1570221eee8c9739f965bb4874bad44ee4c3373a5860d24c90/nvidia_cuda_crt-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee2ea2a97073e02ee62bb27841f437332be2c248e3eac013df07997ada39c003", size = 134086, upload-time = "2025-09-04T08:25:33.024Z" }, + { url = "https://files.pythonhosted.org/packages/05/69/a1ec4d9f0747d85964206feb47bf359f48a2d6af74c0add8abba6efe3dda/nvidia_cuda_crt-13.0.88-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2c8043c7c9e02492716426e9919fc78d2c5b3b2a7a768a88e952676b08aa55a4", size = 134086, upload-time = "2025-09-04T08:25:40.741Z" }, ] [[package]] @@ -3839,7 +3878,7 @@ wheels = [ [[package]] name = "nvidia-cuda-nvcc" -version = "13.3.73" +version = "13.0.88" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nvidia-cuda-crt" }, @@ -3847,8 +3886,8 @@ dependencies = [ { name = "nvidia-nvvm" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/14/9f5cdc994d5431e2f08f62ffe34509e7feabd1f2e18517e2d7720c6ff0fd/nvidia_cuda_nvcc-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:70f250825355d2c3aa6c7a972a0ec00f020bad66d2679e527eb4336301c904aa", size = 39515578, upload-time = "2026-06-29T16:47:40.318Z" }, - { url = "https://files.pythonhosted.org/packages/83/19/e46ef3597ba47a9f8a91ab24533db42a600b659fc418dbe4af0b630bcb41/nvidia_cuda_nvcc-13.3.73-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f483af83166c4fa356a21606076d553b0b4ceaebbd9912e537545080db695bdd", size = 44942138, upload-time = "2026-06-29T16:48:13.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/06/996d5cdc5ea45fb4a6111a1be4f0caf6556c0cb1bf9684a7252d8771797a/nvidia_cuda_nvcc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c7ff28f86a24effdc6c034fa15230c549a273e4771b10a7fec14996f8cf3307f", size = 32455430, upload-time = "2025-09-04T08:27:27.39Z" }, + { url = "https://files.pythonhosted.org/packages/71/8b/a546c12881fffeba927d810598987df25d74b8b241788c7db8dfc93b0173/nvidia_cuda_nvcc-13.0.88-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:56fe502eb77625a12f25172caa3cdddb4e4c8ba2c8c17dba44b164761b380f03", size = 37384532, upload-time = "2025-09-04T08:27:37.916Z" }, ] [[package]] @@ -3880,14 +3919,14 @@ wheels = [ [[package]] name = "nvidia-cudnn-cu13" -version = "9.19.0.56" +version = "9.20.0.48" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nvidia-cublas" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, - { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321, upload-time = "2026-02-03T20:44:52.837Z" }, + { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, + { url = "https://files.pythonhosted.org/packages/6e/5e/edb9c0ae051602c3ccaffe424256463636d639e27d7f302dde9975ef9e7a/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0c45dd8eeb50b603f07995b1b300c62ffe6a1980482b82b3bcf94a4ca9d49304", size = 366173588, upload-time = "2026-03-09T19:29:34.474Z" }, ] [[package]] @@ -3965,11 +4004,11 @@ wheels = [ [[package]] name = "nvidia-cusparselt-cu13" -version = "0.8.0" +version = "0.8.1" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/46/10/8dcd1175260706a2fc92a16a52e306b71d4c1ea0b0cc4a9484183399818a/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:400c6ed1cf6780fc6efedd64ec9f1345871767e6a1a0a552a1ea0578117ea77c", size = 220791277, upload-time = "2025-08-13T19:22:40.982Z" }, - { url = "https://files.pythonhosted.org/packages/fd/53/43b0d71f4e702fa9733f8b4571fdca50a8813f1e450b656c239beff12315/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25e30a8a7323935d4ad0340b95a0b69926eee755767e8e0b1cf8dd85b197d3fd", size = 169884119, upload-time = "2025-08-13T19:23:41.967Z" }, + { url = "https://files.pythonhosted.org/packages/46/e1/cdc1797eadf82d3a9a575a19b33fdc871a97edbec42c00b5b5e914f4aff4/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_aarch64.whl", hash = "sha256:4dca476c50bf4780d46cd0bfbd82e2bc10a08e4fef7950917ce8d7578d22a23f", size = 221051344, upload-time = "2025-09-05T18:49:51.289Z" }, + { url = "https://files.pythonhosted.org/packages/34/7d/2661f2fb3ac4302f3a246f5fc030213ac60c1fe0bce84f9783dbd831dbb7/nvidia_cusparselt_cu13-0.8.1-py3-none-manylinux2014_x86_64.whl", hash = "sha256:786ce87568c303fadb5afcc7102d454cd3040d75f6f8626f5db460d1871f4dd0", size = 170148586, upload-time = "2025-09-05T18:50:50.248Z" }, ] [[package]] @@ -4094,11 +4133,11 @@ wheels = [ [[package]] name = "nvidia-nccl-cu13" -version = "2.28.9" +version = "2.29.7" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/55/1920646a2e43ffd4fc958536b276197ed740e9e0c54105b4bb3521591fc7/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:01c873ba1626b54caa12272ed228dc5b2781545e0ae8ba3f432a8ef1c6d78643", size = 196561677, upload-time = "2025-11-18T05:49:03.45Z" }, - { url = "https://files.pythonhosted.org/packages/b0/b4/878fefaad5b2bcc6fcf8d474a25e3e3774bc5133e4b58adff4d0bca238bc/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:e4553a30f34195f3fa1da02a6da3d6337d28f2003943aa0a3d247bbc25fefc42", size = 196493177, upload-time = "2025-11-18T05:49:17.677Z" }, + { url = "https://files.pythonhosted.org/packages/72/0d/daf50d44177ee0cbc7ff0a0c91eb5ff676c82be42f9a970bc7597f440c3a/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:674a12383e3c38a1bcccae7d4f3633b37852230b6047883cb2f4c2d1b36d9bf5", size = 206014712, upload-time = "2026-03-03T05:34:20.843Z" }, + { url = "https://files.pythonhosted.org/packages/67/f4/58e4e91b6919367c7aafb8e36fce9aad1a3047e536bf7e2fd560927d3a4c/nvidia_nccl_cu13-2.29.7-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:edd81538446786ec3b73972543e53bb43bcaf0bfc8ef76cb679fcc390ffe136d", size = 205976000, upload-time = "2026-03-03T05:36:24.472Z" }, ] [[package]] @@ -4130,11 +4169,11 @@ wheels = [ [[package]] name = "nvidia-nvvm" -version = "13.3.73" +version = "13.0.88" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/e7/ff646aa6015c7e6d12aad234e68925c87b6681d8d18c3ac40535994a3b0d/nvidia_nvvm-13.3.73-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:0e28e0858a3475e11ac67d35301cd5bf82666a1c0dc4ec4e80ceaf3a5fd1dea8", size = 69250424, upload-time = "2026-06-29T17:08:07.453Z" }, - { url = "https://files.pythonhosted.org/packages/2f/05/35754a7105563fd9b496e5ee8e1acd986aef8258760c3cbccf419aee861a/nvidia_nvvm-13.3.73-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e2bcdd5783b5481445f1f0e7170cb836cc0d72999839ba850bbba6dc97b76bb8", size = 66984478, upload-time = "2026-06-29T17:07:43.765Z" }, + { url = "https://files.pythonhosted.org/packages/15/b0/ee41e6d1108d959b5097163e7190c2d0f7857dea75606ce358f0275891b4/nvidia_nvvm-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:c5f41ffeb6466944a026dfa5317d7d85355c119bbec279205d22f1869d1054e0", size = 61601415, upload-time = "2025-09-04T08:37:09.142Z" }, + { url = "https://files.pythonhosted.org/packages/a4/bd/fc52fbf7214391909d6d2b3a825fd0902ebf7fbc56227dd9c9277e8e263b/nvidia_nvvm-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c4376a291d72d22a315d9d2f69bdae8f8cd83a627f75bad395cee49a0fe65dc1", size = 59234185, upload-time = "2025-09-04T08:36:46.505Z" }, ] [[package]] @@ -5524,7 +5563,7 @@ wheels = [ [[package]] name = "quack-kernels" -version = "0.6.3" +version = "0.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-tvm-ffi" }, @@ -5533,9 +5572,9 @@ dependencies = [ { name = "torch" }, { name = "torch-c-dlpack-ext" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/87/6e/589b7e1ac366eaf2f526e350ce9785200f5ef021ad93a92823c5c12cbefc/quack_kernels-0.6.3.tar.gz", hash = "sha256:e307269931e18590f7555afb55debf28e266b4371960aad0528e4055ca8c79a3", size = 839016, upload-time = "2026-08-06T10:28:04.657Z" } +sdist = { url = "https://files.pythonhosted.org/packages/01/17/890875f88f4d7da28faec9e6cf0a0cc565715b01474e50501fafc5bc71b4/quack_kernels-0.6.1.tar.gz", hash = "sha256:a694f89c91d137478de523c0227365a331ac9cb66790cfb08baa3dbfaafc71e7", size = 387353, upload-time = "2026-07-05T11:50:19.807Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/7c/c16475d19327376f2d5d64ada9f8eff0077fce0b54fd87408b4bb0c04b49/quack_kernels-0.6.3-py3-none-any.whl", hash = "sha256:dfa69468a3f71fb9f7192c087c82e85ad68d59155fa03e408a19c7069fb21b8f", size = 727601, upload-time = "2026-08-06T10:28:03.325Z" }, + { url = "https://files.pythonhosted.org/packages/2b/65/a38a30a6ac96a757363a5be9d09cef799640bb143a64ba5a2f4d400d95d9/quack_kernels-0.6.1-py3-none-any.whl", hash = "sha256:266705ea82117e9b1c8a9e44d68a458519f2498d966c0efffd6812120c3995ad", size = 358439, upload-time = "2026-07-05T11:50:18.502Z" }, ] [[package]] @@ -6376,7 +6415,7 @@ wheels = [ [[package]] name = "tilelang" -version = "0.1.9" +version = "0.1.12" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "apache-tvm-ffi" }, @@ -6391,10 +6430,10 @@ dependencies = [ { name = "typing-extensions" }, { name = "z3-solver" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/56/70/5051f65821baa30a3d61fc48f8ba10c776490315e8c90f82559b92089756/tilelang-0.1.9.tar.gz", hash = "sha256:287f727c913bb648fcf6c1968809ba3390e55eeed257a5c6bb9a80bc05966af4", size = 93395292, upload-time = "2026-04-22T09:19:11.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6d/9a/5776adb5b6e03e141c8cff020e16c45b87c9b7a7a5bdfe83416d6cc23933/tilelang-0.1.12.tar.gz", hash = "sha256:595b39581d099a53f875bd8d553863e7e4f58998052b58764f5472cbfbb87043", size = 93565090, upload-time = "2026-07-08T04:22:52.901Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f7/8a/1cbeee79d62abaa02441c2d00621554e41aa62dbf3b94a4feb3867184b01/tilelang-0.1.9-cp38-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bbccfe9035aed775ffafb6dc25a5994504b24e2c5d95d0f39643edfafa7bf12", size = 45419374, upload-time = "2026-04-22T09:15:56.014Z" }, - { url = "https://files.pythonhosted.org/packages/c6/a7/f4bfb86f87e107703146e703204cec2c0eae2492b633e0052b0ace3febb6/tilelang-0.1.9-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:77ab0ee2f40f66ea015b6b21426d482751e28cbc635ef9d1198cbd6502454a7c", size = 42110365, upload-time = "2026-04-22T09:17:18.292Z" }, + { url = "https://files.pythonhosted.org/packages/d1/53/f281a0bd9ee7e03d6a97828fc0e443321ed26ea2b0bd74bf9f1d9451d30f/tilelang-0.1.12-cp38-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bbeb5573cbe2544a51a5c9a6cc737c5e3b5c2d95411caa3687fd9b08eb9d5f97", size = 50501074, upload-time = "2026-07-08T04:22:39.973Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a4/ff0b14d00d29fe418063cd97490f55aa9cad6f981a6fbbf29124b024637d/tilelang-0.1.12-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:a1701eba4ad603d73f1b128b88f6a2350f1b1161729e5b2d1e540a1ec7a82d52", size = 46281933, upload-time = "2026-07-08T04:22:44.244Z" }, ] [[package]] @@ -6512,10 +6551,10 @@ wheels = [ [[package]] name = "torch" -version = "2.11.0" +version = "2.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-bindings" }, + { name = "cuda-bindings", marker = "python_full_version < '3.15'" }, { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"] }, { name = "filelock" }, { name = "fsspec" }, @@ -6527,22 +6566,20 @@ dependencies = [ { name = "nvidia-nvshmem-cu13" }, { name = "setuptools" }, { name = "sympy" }, - { name = "triton" }, + { name = "triton", marker = "python_full_version < '3.15'" }, { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/84/03/acea680005f098f79fd70c1d9d5ccc0cb4296ec2af539a0450108232fc0c/torch-2.11.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:d91aac77f24082809d2c5a93f52a5f085032740a1ebc9252a7b052ef5a4fddc6", size = 419718178, upload-time = "2026-03-23T18:10:46.675Z" }, - { url = "https://files.pythonhosted.org/packages/8c/8b/d7be22fbec9ffee6cff31a39f8750d4b3a65d349a286cf4aec74c2375662/torch-2.11.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7aa2f9bbc6d4595ba72138026b2074be1233186150e9292865e04b7a63b8c67a", size = 530604548, upload-time = "2026-03-23T18:10:03.569Z" }, - { url = "https://files.pythonhosted.org/packages/13/16/42e5915ebe4868caa6bac83a8ed59db57f12e9a61b7d749d584776ed53d5/torch-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f99924682ef0aa6a4ab3b1b76f40dc6e273fca09f367d15a524266db100a723f", size = 419731115, upload-time = "2026-03-23T18:11:06.944Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c9/82638ef24d7877510f83baf821f5619a61b45568ce21c0a87a91576510aa/torch-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0f68f4ac6d95d12e896c3b7a912b5871619542ec54d3649cf48cc1edd4dd2756", size = 530712279, upload-time = "2026-03-23T18:10:31.481Z" }, - { url = "https://files.pythonhosted.org/packages/32/d1/8ed2173589cbfe744ed54e5a73efc107c0085ba5777ee93a5f4c1ab90553/torch-2.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:63a68fa59de8f87acc7e85a5478bb2dddbb3392b7593ec3e78827c793c4b73fd", size = 419732382, upload-time = "2026-03-23T18:08:30.835Z" }, - { url = "https://files.pythonhosted.org/packages/3d/e1/b73f7c575a4b8f87a5928f50a1e35416b5e27295d8be9397d5293e7e8d4c/torch-2.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cc89b9b173d9adfab59fd227f0ab5e5516d9a52b658ae41d64e59d2e55a418db", size = 530711509, upload-time = "2026-03-23T18:08:47.213Z" }, - { url = "https://files.pythonhosted.org/packages/6d/6c/56bfb37073e7136e6dd86bfc6af7339946dd684e0ecf2155ac0eee687ae1/torch-2.11.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:2658f34ce7e2dabf4ec73b45e2ca68aedad7a5be87ea756ad656eaf32bf1e1ea", size = 419732324, upload-time = "2026-03-23T18:09:36.604Z" }, - { url = "https://files.pythonhosted.org/packages/07/f4/1b666b6d61d3394cca306ea543ed03a64aad0a201b6cd159f1d41010aeb1/torch-2.11.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:98bb213c3084cfe176302949bdc360074b18a9da7ab59ef2edc9d9f742504778", size = 530596026, upload-time = "2026-03-23T18:09:20.842Z" }, - { url = "https://files.pythonhosted.org/packages/c7/86/7cd7c66cb9cec6be330fff36db5bd0eef386d80c031b581ec81be1d4b26c/torch-2.11.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:2bb3cc54bd0dea126b0060bb1ec9de0f9c7f7342d93d436646516b0330cd5be7", size = 419749385, upload-time = "2026-03-23T18:07:33.77Z" }, - { url = "https://files.pythonhosted.org/packages/47/e8/b98ca2d39b2e0e4730c0ee52537e488e7008025bc77ca89552ff91021f7c/torch-2.11.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4dc8b3809469b6c30b411bb8c4cad3828efd26236153d9beb6a3ec500f211a60", size = 530716756, upload-time = "2026-03-23T18:07:50.02Z" }, - { url = "https://files.pythonhosted.org/packages/fd/66/54a56a4a6ceaffb567231994a9745821d3af922a854ed33b0b3a278e0a99/torch-2.11.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:ab9a8482f475f9ba20e12db84b0e55e2f58784bdca43a854a6ccd3fd4b9f75e6", size = 419735835, upload-time = "2026-03-23T18:07:18.974Z" }, - { url = "https://files.pythonhosted.org/packages/b1/e7/0b6665f533aa9e337662dc190425abc0af1fe3234088f4454c52393ded61/torch-2.11.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:563ed3d25542d7e7bbc5b235ccfacfeb97fb470c7fee257eae599adb8005c8a2", size = 530613405, upload-time = "2026-03-23T18:08:07.014Z" }, + { url = "https://files.pythonhosted.org/packages/c2/59/1e3160e18e12aa3038390efab3ce02b36a9d4d6a527ecdd8520dca2e68d8/torch-2.13.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:092790c696a760c729fd5722835f50b9d81fd7c8f141571f3f3cf4081a8f664c", size = 427199369, upload-time = "2026-07-08T16:04:51.054Z" }, + { url = "https://files.pythonhosted.org/packages/01/79/1f2d34ad7034ee1c7ffc1cf8bf0f8213af2a81df6ecdb3997ecec107c09d/torch-2.13.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:60fcdcb2f3876e21146cb4524ef06397d727ca9ad5f020818547e25075fe3cb7", size = 526574961, upload-time = "2026-07-08T16:04:07.075Z" }, + { url = "https://files.pythonhosted.org/packages/df/a9/f6a2a4d763ff1df02e9a64c477029db614295bc9367f4131223791ccc243/torch-2.13.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:572df8be8ffb4599c88cbd6a0726f1f854f4da65d2e3c09f0e2c2283333cd6d4", size = 427210998, upload-time = "2026-07-08T16:04:37.708Z" }, + { url = "https://files.pythonhosted.org/packages/f3/82/fea946351658e6534db52d2cc12bc53087cbf87f9440c5f180f367c1950b/torch-2.13.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:796633c4cdf0fe2cdced72d8f88f22e73dbcfce83132763162f6d4bff13b820b", size = 526605292, upload-time = "2026-07-08T16:04:22.81Z" }, + { url = "https://files.pythonhosted.org/packages/11/18/9ecb37b56293a0be8d80f810bf672a72fe7e02f8b475d5ef1b9bf8a0d748/torch-2.13.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:1e09d6a722504957c694faceca843acde562786df1144ebcc5a74075ec7f6005", size = 427213008, upload-time = "2026-07-08T16:03:44.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/5a/7c50ba1b7b713d71d34669c6d13dab0a11531a3eceb0307a5162dbfec0f7/torch-2.13.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:a3a9a21312872af8a26950b2c15680335a386a1f56ed03e780653d78b9607e9e", size = 526602329, upload-time = "2026-07-08T16:03:12.649Z" }, + { url = "https://files.pythonhosted.org/packages/e9/09/2c10e8cd0e00fa5d23c052df6ce467eaa7182399f5e0f824f1e4ff42ccae/torch-2.13.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a3893dc2da0a972a8ca5d698c85a9f967559ac5f8ee1797b77408aa8734d073c", size = 427226309, upload-time = "2026-07-08T16:02:53.127Z" }, + { url = "https://files.pythonhosted.org/packages/76/c6/22c2102bbef14ca6a6cb4c20e42f088e49c5f812be4e160ae57502e325f9/torch-2.13.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:49f1ea385c754e54919408a9bb3b5a72b0b755bbe2c916c1d6f70afbec4908a2", size = 526614507, upload-time = "2026-07-08T16:02:16.441Z" }, + { url = "https://files.pythonhosted.org/packages/69/d1/491e3a0389430946145888b0203f2b6a759ce2a61481b96a85c2da4f2ced/torch-2.13.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:31061ff56ed8fbf26c749806905aeb749ebeb819810fd5d52508aa5afd90dddc", size = 427219769, upload-time = "2026-07-08T16:02:31.18Z" }, + { url = "https://files.pythonhosted.org/packages/9a/1d/38006e045bf0a1fc28ef01e757c554e59e59a8770c284bc4f47b14e60441/torch-2.13.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:cc26eead4cf51d0b544e31e364dcf000846549c273bd148936fe9d24d29acb92", size = 526571320, upload-time = "2026-07-08T16:01:59.348Z" }, ] [[package]] @@ -6602,7 +6639,7 @@ wheels = [ [[package]] name = "torchvision" -version = "0.26.0" +version = "0.28.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, @@ -6611,18 +6648,16 @@ dependencies = [ { name = "torch" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/33/bf/21b899792b08cae7a298551c68398a79e333697479ed311b3b067aab4bdc/torchvision-0.26.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:1c55dc8affbcc0eb2060fbabbe996ae9e5839b24bb6419777f17848945a411b1", size = 7767527, upload-time = "2026-03-23T18:12:44.348Z" }, - { url = "https://files.pythonhosted.org/packages/9a/45/57bbf9e216850d065e66dd31a50f57424b607f1d878ab8956e56a1f4e36b/torchvision-0.26.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:fd10b5f994c210f4f6d6761cf686f82d748554adf486cb0979770c3252868c8f", size = 7519925, upload-time = "2026-03-23T18:12:53.283Z" }, - { url = "https://files.pythonhosted.org/packages/f4/ec/5c31c92c08b65662fe9604a4067ae8232582805949f11ddc042cebe818ed/torchvision-0.26.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:406557718e62fdf10f5706e88d8a5ec000f872da913bf629aab9297622585547", size = 7767944, upload-time = "2026-03-23T18:12:42.805Z" }, - { url = "https://files.pythonhosted.org/packages/f5/d8/cb6ccda1a1f35a6597645818641701207b3e8e13553e75fce5d86bac74b2/torchvision-0.26.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d61a5abb6b42a0c0c311996c2ac4b83a94418a97182c83b055a2a4ae985e05aa", size = 7522205, upload-time = "2026-03-23T18:12:54.654Z" }, - { url = "https://files.pythonhosted.org/packages/e6/81/0b3e58d1478c660a5af4268713486b2df7203f35abd9195fea87348a5178/torchvision-0.26.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a39c7a26538c41fda453f9a9692b5ff9b35a5437db1d94f3027f6f509c160eac", size = 7727494, upload-time = "2026-03-23T18:12:46.062Z" }, - { url = "https://files.pythonhosted.org/packages/b6/dc/d9ab5d29115aa05e12e30f1397a3eeae1d88a511241dc3bce48dc4342675/torchvision-0.26.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:b7e6213620bbf97742e5f79832f9e9d769e6cf0f744c5b53dad80b76db633691", size = 7521747, upload-time = "2026-03-23T18:12:36.815Z" }, - { url = "https://files.pythonhosted.org/packages/71/e2/7a89096e6cf2f3336353b5338ba925e0addf9d8601920340e6bdf47e8eb3/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:3daf9cc149cf3cdcbd4df9c59dae69ffca86c6823250442c3bbfd63fc2e26c61", size = 7728679, upload-time = "2026-03-23T18:12:26.196Z" }, - { url = "https://files.pythonhosted.org/packages/69/1d/4e1eebc17d18ce080a11dcf3df3f8f717f0efdfa00983f06e8ba79259f61/torchvision-0.26.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:82c3965eca27e86a316e31e4c3e5a16d353e0bcbe0ef8efa2e66502c54493c4b", size = 7609138, upload-time = "2026-03-23T18:12:35.327Z" }, - { url = "https://files.pythonhosted.org/packages/7b/ac/48f28ffd227991f2e14f4392dde7e8dc14352bb9428c1ef4a4bbf5f7ed85/torchvision-0.26.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:9a904f2131cbfadab4df828088a9f66291ad33f49ff853872aed1f86848ef776", size = 7727777, upload-time = "2026-03-23T18:12:22.549Z" }, - { url = "https://files.pythonhosted.org/packages/a4/21/a2266f7f1b0e58e624ff15fd6f01041f59182c49551ece0db9a183071329/torchvision-0.26.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:0f3e572efe62ad645017ea847e0b5e4f2f638d4e39f05bc011d1eb9ac68d4806", size = 7522174, upload-time = "2026-03-23T18:12:29.565Z" }, - { url = "https://files.pythonhosted.org/packages/d2/6a/18a582fe3c5ee26f49b5c9fb21ad8016b4d1c06d10178894a58653946fda/torchvision-0.26.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:7058c5878262937e876f20c25867b33724586aa4499e2853b2d52b99a5e51953", size = 7729089, upload-time = "2026-03-23T18:12:31.394Z" }, - { url = "https://files.pythonhosted.org/packages/c5/9b/f7e119b59499edc00c55c03adc9ec3bd96144d9b81c46852c431f9c64a9a/torchvision-0.26.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:8008474855623c6ba52876589dc52df0aa66e518c25eca841445348e5f79844c", size = 7522704, upload-time = "2026-03-23T18:12:20.301Z" }, + { url = "https://files.pythonhosted.org/packages/27/be/1b9c5de9c655ca2df4a74100fa671a7b848532ff787e077ccde14a7dea2a/torchvision-0.28.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:5a38bc6da3d72621be003400b66f66a2b4c6d644fde05f680c2cb7ca8cf8dd6c", size = 7841822, upload-time = "2026-07-08T16:07:49.207Z" }, + { url = "https://files.pythonhosted.org/packages/0b/9b/f1e68e861d4462e3e195a642c2b448e7b7d3fad5f209487162b9a2133d9b/torchvision-0.28.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:7e80f543b22503d9415e126db5f0ff3917036925e38560ee6b9ae38c571a4002", size = 7670718, upload-time = "2026-07-08T16:07:46.525Z" }, + { url = "https://files.pythonhosted.org/packages/f0/4c/95233776e2def960e5abb7a07931230a545f43717a56a1e1140162033598/torchvision-0.28.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5cf78ebc401ce64ae19b8c55de866bb836797d559a4de9c25ccbe74cfa642d3a", size = 7842127, upload-time = "2026-07-08T16:07:53.446Z" }, + { url = "https://files.pythonhosted.org/packages/93/e4/e9b2495d0d57b9f60d63c57d0a910410a81b4b073bf70917bef815291119/torchvision-0.28.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:028a3d481b37d785605620d7cdad897064c5a55bae2aa1f2658766333e291940", size = 7675040, upload-time = "2026-07-08T16:07:58.017Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/40beacd53809194f5259e590d1afaeaa8ad57da15f77c646e6560bcc4616/torchvision-0.28.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:bb6dd6918460ed89cc7644adcc2402991474d6933cf1ce92b390641cb233fddf", size = 7797014, upload-time = "2026-07-08T16:07:43.04Z" }, + { url = "https://files.pythonhosted.org/packages/32/db/062cdb5a84380a60439775311fff34d89229760d2a50680393dc18699956/torchvision-0.28.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:ad7b3a439265cc3739a4ab5b4c998c0e38ea99c0ee7ca4dea35c5d0b099ec237", size = 7674669, upload-time = "2026-07-08T16:07:38.91Z" }, + { url = "https://files.pythonhosted.org/packages/06/d6/313aafd3df4eaf5f330211bd4e75b7598bddbfee4f55580d3b58536e1b20/torchvision-0.28.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:89f90e29b0966352811b12589f3a3c61943bf2bb9487b9d7bbec10efb1096bb5", size = 7796873, upload-time = "2026-07-08T16:07:30.907Z" }, + { url = "https://files.pythonhosted.org/packages/b3/41/31f8e959ab8f942600b6357f8999c21d779d5fd3304b0fd204ff4b518239/torchvision-0.28.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:36beb0782976906069ca03d4c9aacaf4b6b838b06ed6c20960ea9c51cce7acdd", size = 7674634, upload-time = "2026-07-08T16:07:29.657Z" }, + { url = "https://files.pythonhosted.org/packages/7f/d1/cd3f9463b39a790ec8c0c2f6e6c8061edb1562114d04fcdfa786ed889345/torchvision-0.28.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:62c7d110f86a039245b587e4fae60278c649f3bd42ff79cfbc1178eca4e72542", size = 7796742, upload-time = "2026-07-08T16:07:28.339Z" }, + { url = "https://files.pythonhosted.org/packages/d9/82/3e0a7ad18e99831e2d7f4713d3be717b7159ff5a920862dd5c23c454aa71/torchvision-0.28.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:904cf89af220f8c6b2ed0296bb5065b474ce43b77558e48b2bf9de8b0ba17204", size = 7675526, upload-time = "2026-07-08T16:07:34.572Z" }, ] [[package]] @@ -6686,21 +6721,19 @@ wheels = [ [[package]] name = "triton" -version = "3.6.0" +version = "3.7.1" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/2c/96f92f3c60387e14cc45aed49487f3486f89ea27106c1b1376913c62abe4/triton-3.6.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:49df5ef37379c0c2b5c0012286f80174fcf0e073e5ade1ca9a86c36814553651", size = 176081190, upload-time = "2026-01-20T16:16:00.523Z" }, - { url = "https://files.pythonhosted.org/packages/e0/12/b05ba554d2c623bffa59922b94b0775673de251f468a9609bc9e45de95e9/triton-3.6.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8e323d608e3a9bfcc2d9efcc90ceefb764a82b99dea12a86d643c72539ad5d3", size = 188214640, upload-time = "2026-01-20T16:00:35.869Z" }, - { url = "https://files.pythonhosted.org/packages/17/5d/08201db32823bdf77a0e2b9039540080b2e5c23a20706ddba942924ebcd6/triton-3.6.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4", size = 176128243, upload-time = "2026-01-20T16:16:07.857Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a8/cdf8b3e4c98132f965f88c2313a4b493266832ad47fb52f23d14d4f86bb5/triton-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca", size = 188266850, upload-time = "2026-01-20T16:00:43.041Z" }, - { url = "https://files.pythonhosted.org/packages/3c/12/34d71b350e89a204c2c7777a9bba0dcf2f19a5bfdd70b57c4dbc5ffd7154/triton-3.6.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448e02fe6dc898e9e5aa89cf0ee5c371e99df5aa5e8ad976a80b93334f3494fd", size = 176133521, upload-time = "2026-01-20T16:16:13.321Z" }, - { url = "https://files.pythonhosted.org/packages/f9/0b/37d991d8c130ce81a8728ae3c25b6e60935838e9be1b58791f5997b24a54/triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c7f76c6e72d2ef08df639e3d0d30729112f47a56b0c81672edc05ee5116ac9", size = 188289450, upload-time = "2026-01-20T16:00:49.136Z" }, - { url = "https://files.pythonhosted.org/packages/ce/4e/41b0c8033b503fd3cfcd12392cdd256945026a91ff02452bef40ec34bee7/triton-3.6.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1722e172d34e32abc3eb7711d0025bb69d7959ebea84e3b7f7a341cd7ed694d6", size = 176276087, upload-time = "2026-01-20T16:16:18.989Z" }, - { url = "https://files.pythonhosted.org/packages/35/f8/9c66bfc55361ec6d0e4040a0337fb5924ceb23de4648b8a81ae9d33b2b38/triton-3.6.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d002e07d7180fd65e622134fbd980c9a3d4211fb85224b56a0a0efbd422ab72f", size = 188400296, upload-time = "2026-01-20T16:00:56.042Z" }, - { url = "https://files.pythonhosted.org/packages/49/55/5ecf0dcaa0f2fbbd4420f7ef227ee3cb172e91e5fede9d0ecaddc43363b4/triton-3.6.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5523241e7d1abca00f1d240949eebdd7c673b005edbbce0aca95b8191f1d43", size = 176138577, upload-time = "2026-01-20T16:16:25.426Z" }, - { url = "https://files.pythonhosted.org/packages/df/3d/9e7eee57b37c80cec63322c0231bb6da3cfe535a91d7a4d64896fcb89357/triton-3.6.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a17a5d5985f0ac494ed8a8e54568f092f7057ef60e1b0fa09d3fd1512064e803", size = 188273063, upload-time = "2026-01-20T16:01:07.278Z" }, - { url = "https://files.pythonhosted.org/packages/48/db/56ee649cab5eaff4757541325aca81f52d02d4a7cd3506776cad2451e060/triton-3.6.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b3a97e8ed304dfa9bd23bb41ca04cdf6b2e617d5e782a8653d616037a5d537d", size = 176274804, upload-time = "2026-01-20T16:16:31.528Z" }, - { url = "https://files.pythonhosted.org/packages/f6/56/6113c23ff46c00aae423333eb58b3e60bdfe9179d542781955a5e1514cb3/triton-3.6.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46bd1c1af4b6704e554cad2eeb3b0a6513a980d470ccfa63189737340c7746a7", size = 188397994, upload-time = "2026-01-20T16:01:14.236Z" }, + { url = "https://files.pythonhosted.org/packages/7b/f9/19d842d06a08559534fa1eaab6ca551b1bcf40f06620bddec1babaa2772d/triton-3.7.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4a0e1cd4c4a76370ed74a8432a53cea28716827d19e40ffc732233e35ceb3f6", size = 184664887, upload-time = "2026-06-17T20:03:42.913Z" }, + { url = "https://files.pythonhosted.org/packages/cd/5e/fce69606f7f240297f163e25539906732b199530d486ce67ae319877e821/triton-3.7.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6744957e9fd610a29680ec2346057d0c86948ed3812468670719f391e94b44a5", size = 197701306, upload-time = "2026-06-17T19:53:13.673Z" }, + { url = "https://files.pythonhosted.org/packages/94/fa/f856e24deb462d5f18bd4b5a746957862ab9b6ee5834bda60605ec348366/triton-3.7.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9497f2e696ee368862a181a90b2dcc03ca978cc4f602abd67c7d81022a6988e1", size = 184692359, upload-time = "2026-06-17T20:03:48.288Z" }, + { url = "https://files.pythonhosted.org/packages/c4/6f/fb96d15db6f36d6eae4cafb998c2e0353bf59d7c4ea1662d7497f269134a/triton-3.7.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e40869937a68206ec70d7f25bb7ec6433cb083f9135e1f36dbd318dc449a728", size = 197719725, upload-time = "2026-06-17T19:53:20.419Z" }, + { url = "https://files.pythonhosted.org/packages/00/42/c5089d4d9327fcd1e862c599cc2927f39418f84dd11a84cb2ccff9d4787a/triton-3.7.1-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cdbfc09d9ec58bc5e68321525653220de7515c199e7a8097a97c85e62b52cd0a", size = 184694629, upload-time = "2026-06-17T20:03:53.444Z" }, + { url = "https://files.pythonhosted.org/packages/07/42/2c3ac59253ae8892b6f307875263dd23dc875cdf732d3aea40d6d41fb7cb/triton-3.7.1-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58c0e131da05134a2a4788ccbcc0c1105cf0f54c8e98f19e34cd465396dc15eb", size = 197729241, upload-time = "2026-06-17T19:53:27.801Z" }, + { url = "https://files.pythonhosted.org/packages/40/71/e01aa7ad573883ed9456f130226babdec70b005e098c4d6226a6238e761b/triton-3.7.1-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe4ea396a06171f1f1f58cbd39c70b09294398f7dd7c620939bab54ad6f934fa", size = 184705764, upload-time = "2026-06-17T20:03:59.064Z" }, + { url = "https://files.pythonhosted.org/packages/a4/09/5683146fda6a2b569deb78ccfd8fbfea8bfe55f726b081c0a6bb18dd6f28/triton-3.7.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2020153b08280415ec0da6607834e79166442147e78e144df06b508c75b186d2", size = 197729537, upload-time = "2026-06-17T19:53:35.516Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f8/448220c3092019f9fdfab39ec47985968181d67da34b44f6a7f6280a5cbb/triton-3.7.1-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c58e4c61f0c73b5dba3b5d19b4a7093c32f90dc18b2a7f121a7c16ccd31107b7", size = 184814760, upload-time = "2026-06-17T20:04:04.984Z" }, + { url = "https://files.pythonhosted.org/packages/f0/ac/229b7d4589d2e5937310e72c6d46e89599d16a4a12b479ffa1499fee8eb8/triton-3.7.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10ba85fa2cca4a2fbdeb36bf1cb082f2c252bda55bf9fccd74f65ec5bc647e68", size = 197824404, upload-time = "2026-06-17T19:53:42.772Z" }, ] [[package]] @@ -6877,7 +6910,7 @@ wheels = [ [[package]] name = "vllm" -version = "0.26.0" +version = "0.27.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -6938,8 +6971,8 @@ dependencies = [ { name = "safetensors" }, { name = "sentencepiece" }, { name = "setproctitle" }, - { name = "setuptools", marker = "python_full_version >= '3.12'" }, - { name = "six", marker = "python_full_version >= '3.12'" }, + { name = "setuptools" }, + { name = "six" }, { name = "starlette" }, { name = "tiktoken" }, { name = "tilelang" }, @@ -6955,10 +6988,10 @@ dependencies = [ { name = "watchfiles" }, { name = "xgrammar", marker = "platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'ppc64le' or platform_machine == 's390x' or platform_machine == 'x86_64'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/cd/72/fa30f8459d11ae206f1a20bd0ac7ed1b9e390b695fa3dfc9ef6056de0cfe/vllm-0.26.0.tar.gz", hash = "sha256:23e9fa19d7e20ce7dcc1c074d41503e2116d23f19e688f5d5ea91b741f958502", size = 38353572, upload-time = "2026-07-25T10:40:48.095Z" } +sdist = { url = "https://files.pythonhosted.org/packages/97/fa/a3cb3a7baf9f1f1027580b0542e42f655b0f577b8a8d39878c1c0a73e6a9/vllm-0.27.1.tar.gz", hash = "sha256:eec2d54d137ac1e59cb4c39226dfee1943eefc8f4788f5821d7300d6acbdb646", size = 39257426, upload-time = "2026-08-11T10:59:48.04Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/58/27/6ff13689a5931f0c97b7008042f07aacc4a246e7eb06fd9b4d5a72de483c/vllm-0.26.0-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:52a4c3e55c2c80cc8793e52ccc244457ceade25b0ad7caa1c15e5002a95a1b2c", size = 298269785, upload-time = "2026-07-25T10:40:02.845Z" }, - { url = "https://files.pythonhosted.org/packages/20/96/86edd288415aafc2952bbb969b5ef4e8c58e5525185b60320730276921e6/vllm-0.26.0-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:adb1e4c9b46d0dfdb094121ae5aad670a42412dd813ed4e5db069ed6a15006de", size = 303698761, upload-time = "2026-07-25T10:40:32.107Z" }, + { url = "https://files.pythonhosted.org/packages/ff/be/6d309884d11ef02966c06d633e156d2070b986df6fccef25041c4b9510e2/vllm-0.27.1-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:74c1a04548c9016ace8f6c03592edf62517152a0e833f46282556ffbe0796254", size = 307180998, upload-time = "2026-08-11T11:00:37.201Z" }, + { url = "https://files.pythonhosted.org/packages/8f/73/20ad0cc655a5739d95342075c9afab1a25338d4b584fc71548c0b362404b/vllm-0.27.1-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:98e9fc2a1ed8549a733c9d1b242e2002b82367da9e29e37801761438cb3a2670", size = 312887766, upload-time = "2026-08-11T11:00:11.706Z" }, ] [[package]] From a979cc061b1ec88d2fb8da19d6001d8ba7671286 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Wed, 12 Aug 2026 22:25:06 +0000 Subject: [PATCH 08/28] Add generation model profiles Signed-off-by: Aaron Gonzales --- docs/concepts/inference-services.md | 46 +++++++++++++++++++ tests/tools/test_inference_service.py | 26 +++++++++++ tools/inference_service_compiler/runtime.py | 6 ++- .../gpt-oss-120b.toml | 33 +++++++++++++ .../gpt-oss-20b.toml | 33 +++++++++++++ .../qwen3-30b-a3b-instruct.toml | 33 +++++++++++++ 6 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 tools/inference_service_profiles/gpt-oss-120b.toml create mode 100644 tools/inference_service_profiles/gpt-oss-20b.toml create mode 100644 tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml diff --git a/docs/concepts/inference-services.md b/docs/concepts/inference-services.md index 07a0fd70..4c5c7756 100644 --- a/docs/concepts/inference-services.md +++ b/docs/concepts/inference-services.md @@ -155,6 +155,52 @@ name = "privacy" The compiler renders the corresponding vLLM `--lora-modules` arguments. +### Generation profile selection + +The bundled profiles pin model weights and conservative 8,192-token service +bounds for Anonymizer's structured generation work: + +| Profile | Served model name | Intended use | +| --- | --- | --- | +| `vllm-local.toml` | `anonymizer-local` | Small endpoint and lifecycle smoke tests | +| `gpt-oss-20b.toml` | `gpt-oss-20b-local` | GPT-OSS development on GPUs with at least 16 GB of model capacity | +| `qwen3-30b-a3b-instruct.toml` | `qwen3-30b-a3b-instruct-local` | Multilingual, non-thinking structured generation | +| `nemotron-3.5-lightning.toml` | `nemotron-3.5-lightning-local` | High-throughput structured generation on an 80 GB GPU | +| `gpt-oss-120b.toml` | `gpt-oss-120b-local` | Higher-quality GPT-OSS generation on a dedicated 80 GB GPU | + +Treat these as reproducible starting points. Available memory also depends on +the GPU architecture, driver, context length, concurrency, and other processes. +Reduce `max_model_len` or `max_num_seqs` when vLLM cannot reserve its KV cache. + +### GPT-OSS + +The `gpt-oss-20b.toml` and `gpt-oss-120b.toml` profiles pin OpenAI's native +MXFP4 checkpoints. The 120B profile reserves one 80 GB GPU for generation; do +not co-host GLiNER on that GPU. Both profiles enable prefix caching and +asynchronous scheduling for Anonymizer's repeated structured prompts. + +Compile either profile with the same workflow used for other generation +models: + +```bash +uv run tools/inference_service.py compile \ + --profile tools/inference_service_profiles/gpt-oss-120b.toml \ + --source-revision 3f68c145 \ + --output gpt-oss-120b-plan.json +``` + +GPT-OSS defaults to medium reasoning. For Anonymizer's structured LLM roles, +start with low reasoning by setting +`inference_parameters.extra_body.chat_template_kwargs.reasoning_effort` to +`low`. Keep `max_tokens` within the profile's context bound. + +### Qwen3 30B A3B Instruct + +`qwen3-30b-a3b-instruct.toml` pins the non-thinking July 2025 instruct +checkpoint. It is a practical multilingual alternative when the input is not +primarily English. The profile serves the BF16 checkpoint, so allow roughly +the same single-GPU memory class as the Nemotron Lightning profile. + ### Nemotron 3.5 Lightning `tools/inference_service_profiles/nemotron-3.5-lightning.toml` pins diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index bc7294f7..c3773a93 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -501,8 +501,11 @@ def test_reference_toml_profiles_are_pinned_and_compile() -> None: assert {path.name for path in profile_paths} == { "gliner2.toml", + "gpt-oss-120b.toml", + "gpt-oss-20b.toml", "nemotron-3.5-lightning.toml", "nvidia-gliner.toml", + "qwen3-30b-a3b-instruct.toml", "vllm-local.toml", } for path in profile_paths: @@ -543,6 +546,29 @@ def respond(request: httpx.Request) -> httpx.Response: assert receipt.passed is True +def test_probe_supports_reasoning_models() -> None: + """The capability probe obtains content from models that reason before answering.""" + models, compiler = load_compiler_modules() + runtime = load_runtime_module() + plan = build_generation_plan(models, compiler) + + def respond(request: httpx.Request) -> httpx.Response: + if request.url.path == "/v1/models": + return httpx.Response(200, json={"data": [{"id": "openai/gpt-oss-20b"}]}) + if request.url.path == "/v1/chat/completions": + payload = json.loads(request.content) + enough_output = payload.get("max_tokens", 0) > 8 + low_reasoning = payload.get("chat_template_kwargs", {}).get("reasoning_effort") == "low" + content = "ready" if enough_output and low_reasoning else None + return httpx.Response(200, json={"choices": [{"message": {"content": content}}]}) + return httpx.Response(404) + + with httpx.Client(transport=httpx.MockTransport(respond)) as client: + receipt = runtime.probe_endpoint(plan, client=client) + + assert receipt.passed is True + + def test_probe_uses_the_resolved_bearer_secret() -> None: """Secured readiness and task probes authenticate without serializing the value.""" models, compiler = load_compiler_modules() diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index 9054ed09..09b9d3b4 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -353,7 +353,11 @@ def _probe_task(plan: RunPlan, client: httpx.Client, headers: Mapping[str, str]) json={ "model": plan.expected_model, "messages": [{"role": "user", "content": "Reply with the word ready."}], - "max_tokens": 8, + "max_tokens": 128, + "chat_template_kwargs": { + "enable_thinking": False, + "reasoning_effort": "low", + }, }, headers=headers, ) diff --git a/tools/inference_service_profiles/gpt-oss-120b.toml b/tools/inference_service_profiles/gpt-oss-120b.toml new file mode 100644 index 00000000..b6ae18ef --- /dev/null +++ b/tools/inference_service_profiles/gpt-oss-120b.toml @@ -0,0 +1,33 @@ +schema_version = "inference-service.intent/v1" + +[task] +kind = "generation" +chat = true + +[model] +kind = "hugging-face" +model_id = "openai/gpt-oss-120b" +revision = "b5c939de8f754692c1647ca79fbf85e8c1e70f8a" + +[engine] +kind = "vllm" +python_executable = ".venv/bin/python" +served_model_name = "gpt-oss-120b-local" +gpu_memory_utilization = 0.95 +max_model_len = 8192 +max_num_seqs = 8 +enable_prefix_caching = true +async_scheduling = true + +[placement] +kind = "local-process" +host = "127.0.0.1" +port = 8000 + +[access] +kind = "direct" + +[lifecycle] +kind = "managed" +startup_timeout_seconds = 1800 +shutdown_timeout_seconds = 60 diff --git a/tools/inference_service_profiles/gpt-oss-20b.toml b/tools/inference_service_profiles/gpt-oss-20b.toml new file mode 100644 index 00000000..e3153eb6 --- /dev/null +++ b/tools/inference_service_profiles/gpt-oss-20b.toml @@ -0,0 +1,33 @@ +schema_version = "inference-service.intent/v1" + +[task] +kind = "generation" +chat = true + +[model] +kind = "hugging-face" +model_id = "openai/gpt-oss-20b" +revision = "6cee5e81ee83917806bbde320786a8fb61efebee" + +[engine] +kind = "vllm" +python_executable = ".venv/bin/python" +served_model_name = "gpt-oss-20b-local" +gpu_memory_utilization = 0.85 +max_model_len = 8192 +max_num_seqs = 16 +enable_prefix_caching = true +async_scheduling = true + +[placement] +kind = "local-process" +host = "127.0.0.1" +port = 8000 + +[access] +kind = "direct" + +[lifecycle] +kind = "managed" +startup_timeout_seconds = 1200 +shutdown_timeout_seconds = 60 diff --git a/tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml b/tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml new file mode 100644 index 00000000..d89dfe43 --- /dev/null +++ b/tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml @@ -0,0 +1,33 @@ +schema_version = "inference-service.intent/v1" + +[task] +kind = "generation" +chat = true + +[model] +kind = "hugging-face" +model_id = "Qwen/Qwen3-30B-A3B-Instruct-2507" +revision = "0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe" + +[engine] +kind = "vllm" +python_executable = ".venv/bin/python" +served_model_name = "qwen3-30b-a3b-instruct-local" +gpu_memory_utilization = 0.90 +max_model_len = 8192 +max_num_seqs = 16 +enable_prefix_caching = true +async_scheduling = true + +[placement] +kind = "local-process" +host = "127.0.0.1" +port = 8000 + +[access] +kind = "direct" + +[lifecycle] +kind = "managed" +startup_timeout_seconds = 1800 +shutdown_timeout_seconds = 60 From bf26d87a03b8a13e1c627e3700904158db1fd1f6 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Wed, 12 Aug 2026 23:15:01 +0000 Subject: [PATCH 09/28] Simplify local inference host to vLLM only Signed-off-by: Aaron Gonzales --- .copyrightignore | 2 + README.md | 2 +- docs/concepts/inference-services.md | 331 +----- docs/concepts/models.md | 2 +- docs/concepts/self-hosting-gliner.md | 7 +- tests/tools/test_inference_service.py | 977 ++++-------------- tests/tools/test_native_gliner.py | 284 ----- tools/inference_service_compiler/cli.py | 11 +- tools/inference_service_compiler/compiler.py | 188 +--- tools/inference_service_compiler/models.py | 125 +-- .../native_gliner.py | 681 ------------ tools/inference_service_compiler/runtime.py | 169 +-- tools/inference_service_profiles/gliner2.toml | 16 +- .../gpt-oss-120b.toml | 14 +- .../gpt-oss-20b.toml | 14 +- .../nemotron-3.5-lightning.toml | 14 +- .../nvidia-gliner.toml | 16 +- .../qwen3-30b-a3b-instruct.toml | 14 +- .../vllm-local.toml | 14 +- 19 files changed, 377 insertions(+), 2504 deletions(-) delete mode 100644 tests/tools/test_native_gliner.py delete mode 100755 tools/inference_service_compiler/native_gliner.py diff --git a/.copyrightignore b/.copyrightignore index f72bdbad..7a83eb59 100644 --- a/.copyrightignore +++ b/.copyrightignore @@ -12,3 +12,5 @@ CHANGELOG.md .cursor/ .claude/ .agent/ +skills/anonymizer/BENCHMARK.md +skills/anonymizer/skill-card.md diff --git a/README.md b/README.md index b18253ce..1a631271 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ uv run tools/inference_service.py cancel --receipt launch.json The tool is not part of the wheel and does not attach to externally owned endpoints. The [local inference service guide](docs/concepts/inference-services.md) -covers typed TOML profiles, GPU-host setup, Docker, model discovery, capability +covers typed TOML profiles, GPU-host setup, and capability probes, and Anonymizer provider configuration. --- diff --git a/docs/concepts/inference-services.md b/docs/concepts/inference-services.md index 4c5c7756..13ca7801 100644 --- a/docs/concepts/inference-services.md +++ b/docs/concepts/inference-services.md @@ -3,48 +3,12 @@ # Run Local Inference Services -Anonymizer's source tree includes an inference service compiler for development -and controlled internal deployments. It compiles a typed TOML profile into an -immutable plan before it starts a process or container. Launch, probe, inspect, -and cancel operations return versioned JSON receipts with the external identity -and known effects of the operation. +The source tree provides `tools/inference_service.py` for compiling and managing +one domain: a local-process vLLM server. Compilation is pure and produces a +versioned, digest-protected plan; launch, probe, inspect, and cancel consume that +plan rather than re-reading a profile. -The tool is source-owned under `tools/` and is not included in the -`nemo-anonymizer` wheel. It currently supports: - -- entity detection with NVIDIA GLiNER or GLiNER2 through the external - [vLLM Factory](https://github.com/latenceainew/vllm-factory) project; -- entity detection with the source-owned native runtime when a GPU vLLM stack - is not appropriate; -- generation through a vLLM-compatible model; -- managed local processes and managed local Docker containers; and -- direct HTTP access to the resulting OpenAI-compatible endpoint. - -It does not attach to an existing endpoint or manage remote compute. Configure -an existing provider URL directly in Anonymizer when another system owns that -service. - -## Lifecycle - -The CLI keeps compilation separate from runtime effects: - -```text -profile.toml -> compile -> plan.json -> launch -> launch.json - inspect <-+ - cancel <-+ -``` - -Plans use the `inference-service.run-plan/v1` schema. Launch, capability-probe, -status, and cancellation receipts have their own v1 schemas. A SHA-256 digest -binds each plan to its exact intent, command, endpoint contract, compatibility -evidence, and source revision. Runtime commands reject a changed plan before -performing effects. - -## GLiNER through vLLM Factory - -The pinned NVIDIA GLiNER and GLiNER2 profiles under -`tools/inference_service_profiles/` use vLLM Factory. Install the local model -group on a Linux GPU host: +Install the optional local-model dependencies on a Linux GPU host: ```bash uv sync --python 3.12 --group dev --group local-models @@ -52,259 +16,66 @@ python -m vllm_factory.compat.doctor nvidia-smi ``` -The dependency group pins vLLM 0.27.1 and an exact vLLM Factory source commit. -Local vLLM 0.27.1 serving requires Python 3.12 or later. The group also aligns -the CUDA 13.0 compiler wheels used by FlashInfer's Mamba kernel JIT. -The compiled plan records both dependencies. The runtime calls vLLM Factory's -model-preparation Python API with the profile's pinned Hugging Face revision, -loads its GLiNER model plugin and IOProcessor, then constructs the vLLM server -through vLLM's Python API. It does not invoke either project's CLI. - -Compile and launch the NVIDIA profile from the repository root. Replace the -example source revision with the revision of your checkout: - -```bash -uv run tools/inference_service.py compile \ - --profile tools/inference_service_profiles/nvidia-gliner.toml \ - --source-revision 3f68c145 \ - --output gliner-plan.json - -uv run tools/inference_service.py launch \ - --plan gliner-plan.json \ - --output gliner-launch.json -``` - -The service keeps vLLM Factory's native `POST /pooling` endpoint. A thin -in-process adapter also exposes Anonymizer's `POST /v1/chat/completions` -detector contract. The adapter preserves dynamic labels, character offsets, -scores, overlapping character chunks, and label-free DataDesigner health -checks. Model preparation, scheduling, batching, inference, and decoding stay -inside vLLM Factory and vLLM. - -Launch returns only after `/v1/models` and a positive entity-detection contract -probe succeed. The receipt records the process ID plus its Linux start marker, -when available, which lets a later invocation guard against PID reuse. - -Use `tools/inference_service_profiles/gliner2.toml` for the pinned GLiNER2 -checkpoint and the `deberta_gliner2` plugin. The NVIDIA profile uses -`deberta_gliner`. Stock vLLM remains invalid for entity detection unless the -intent selects one of these characterized factory integrations. - -vLLM Factory detection is characterized as a managed local process. The -compiler rejects its use with the stock vLLM Docker image because that image -does not contain the pinned external project or Anonymizer's protocol adapter. - -## Native GLiNER fallback - -The source-owned native runtime remains available for CPU, MPS, and local GPU -use. Select `kind = "native-gliner"` and choose the family in a custom profile: - -```toml -[engine] -kind = "native-gliner" -family = "nvidia-gliner" # or "gliner2" -device = "auto" -``` - -This path runs `tools/inference_service_compiler/native_gliner.py` as an -isolated uv script. It preserves the same OpenAI-compatible detector contract -but does not use vLLM Factory's scheduler or IOProcessor plugins. - -## Local vLLM Process - -On a Linux GPU host, install the optional source-tree dependency group: - -```bash -uv sync --python 3.12 --group dev --group local-models -nvidia-smi -``` - -The `local-models` group pins vLLM 0.27.1. The local generation plan starts -`tools/inference_service_compiler/vllm_server.py`, which constructs vLLM's -frontend and async engine through its Python API. It does not invoke `vllm -serve` or inherit vLLM's full CLI surface. - -Compile `tools/inference_service_profiles/vllm-local.toml`, or copy it and pin -the model revision and sizing fields for your workload: - -```bash -uv run tools/inference_service.py compile \ - --profile tools/inference_service_profiles/vllm-local.toml \ - --source-revision 3f68c145 \ - --output vllm-plan.json +The group pins vLLM 0.27.1, the external vLLM Factory revision, and the CUDA +compiler wheels needed by the Nemotron profile. -uv run tools/inference_service.py launch \ - --plan vllm-plan.json \ - --output vllm-launch.json -``` +## Profiles -The model ID may cause vLLM to download weights. List existing Hugging Face -cache snapshots without downloading anything: +Seven pinned profiles ship in `tools/inference_service_profiles/`. Generation +profiles use Hugging Face vLLM. `nvidia-gliner.toml` and `gliner2.toml` use the +pinned NVIDIA vLLM Factory integration for entity detection. Detection requires +the Factory section; stock vLLM generation must not include it. -```bash -uv run tools/inference_service.py models --output cached-models.json -``` - -Add a LoRA artifact to the model when needed: - -```toml -[model.adapter] -path = "/models/privacy-adapter" -name = "privacy" -``` - -The compiler renders the corresponding vLLM `--lora-modules` arguments. - -### Generation profile selection - -The bundled profiles pin model weights and conservative 8,192-token service -bounds for Anonymizer's structured generation work: - -| Profile | Served model name | Intended use | +| Profile | Served model | Use | | --- | --- | --- | -| `vllm-local.toml` | `anonymizer-local` | Small endpoint and lifecycle smoke tests | -| `gpt-oss-20b.toml` | `gpt-oss-20b-local` | GPT-OSS development on GPUs with at least 16 GB of model capacity | -| `qwen3-30b-a3b-instruct.toml` | `qwen3-30b-a3b-instruct-local` | Multilingual, non-thinking structured generation | -| `nemotron-3.5-lightning.toml` | `nemotron-3.5-lightning-local` | High-throughput structured generation on an 80 GB GPU | -| `gpt-oss-120b.toml` | `gpt-oss-120b-local` | Higher-quality GPT-OSS generation on a dedicated 80 GB GPU | - -Treat these as reproducible starting points. Available memory also depends on -the GPU architecture, driver, context length, concurrency, and other processes. -Reduce `max_model_len` or `max_num_seqs` when vLLM cannot reserve its KV cache. - -### GPT-OSS - -The `gpt-oss-20b.toml` and `gpt-oss-120b.toml` profiles pin OpenAI's native -MXFP4 checkpoints. The 120B profile reserves one 80 GB GPU for generation; do -not co-host GLiNER on that GPU. Both profiles enable prefix caching and -asynchronous scheduling for Anonymizer's repeated structured prompts. - -Compile either profile with the same workflow used for other generation -models: +| `vllm-local.toml` | `anonymizer-local` | Small lifecycle smoke tests | +| `gpt-oss-20b.toml` | `gpt-oss-20b-local` | Compact GPT-OSS development | +| `gpt-oss-120b.toml` | `gpt-oss-120b-local` | Dedicated 80 GB GPU | +| `qwen3-30b-a3b-instruct.toml` | `qwen3-30b-a3b-instruct-local` | Multilingual generation | +| `nemotron-3.5-lightning.toml` | `nemotron-3.5-lightning-local` | High-throughput generation | +| `nvidia-gliner.toml` | model ID | NVIDIA GLiNER detection | +| `gliner2.toml` | model ID | GLiNER2 detection | + +Memory requirements also depend on the GPU, driver, context length, and +concurrency. Reduce `max_model_len` or `max_num_seqs` if vLLM cannot reserve its +KV cache. Do not co-host the 120B GPT-OSS profile on the GPU used for detection. -```bash -uv run tools/inference_service.py compile \ - --profile tools/inference_service_profiles/gpt-oss-120b.toml \ - --source-revision 3f68c145 \ - --output gpt-oss-120b-plan.json -``` - -GPT-OSS defaults to medium reasoning. For Anonymizer's structured LLM roles, -start with low reasoning by setting -`inference_parameters.extra_body.chat_template_kwargs.reasoning_effort` to -`low`. Keep `max_tokens` within the profile's context bound. - -### Qwen3 30B A3B Instruct - -`qwen3-30b-a3b-instruct.toml` pins the non-thinking July 2025 instruct -checkpoint. It is a practical multilingual alternative when the input is not -primarily English. The profile serves the BF16 checkpoint, so allow roughly -the same single-GPU memory class as the Nemotron Lightning profile. - -### Nemotron 3.5 Lightning - -`tools/inference_service_profiles/nemotron-3.5-lightning.toml` pins -`nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16` and its immutable -Hugging Face revision. The profile uses the model's recommended FlashInfer -Mamba backend, float16 Mamba cache, stochastic cache rounding, asynchronous -scheduling, and prefix caching. It bounds context to 8,192 tokens and -concurrency to 16 so the BF16 model can share one 80 GB GPU with the GLiNER -profile. +## Lifecycle ```bash uv run tools/inference_service.py compile \ - --profile tools/inference_service_profiles/nemotron-3.5-lightning.toml \ - --source-revision 3f68c145 \ - --output lightning-plan.json - -uv run tools/inference_service.py launch \ - --plan lightning-plan.json \ - --output lightning-launch.json -``` - -Lightning enables reasoning by default. For Anonymizer's structured LLM roles, -set `inference_parameters.extra_body.chat_template_kwargs.enable_thinking` to -`false`. Keep `max_tokens` within the profile's context bound; 1,024 tokens is -enough for the detection and replace-evaluation schemas. - -## Docker vLLM generation - -Change the placement to Docker to use vLLM's official OpenAI-compatible image: - -```toml -[placement] -kind = "docker" -host = "127.0.0.1" -port = 8000 -image = "vllm/vllm-openai:v0.27.1" -runtime = "docker" -gpus = "all" -hugging_face_cache = "/home/user/.cache/huggingface" -``` - -The plan records the exact image and complete `docker run` argv. Launch receipts -record the container ID, and `inspect` and `cancel` reconnect through that ID. -Pin an image version appropriate for the host's driver and CUDA compatibility. - -## Secrets - -Set `api_key_env` on the vLLM engine to reference a named environment variable: - -```toml -[engine] -kind = "vllm" -api_key_env = "LOCAL_VLLM_API_KEY" + --profile tools/inference_service_profiles/nvidia-gliner.toml \ + --source-revision "$(git rev-parse HEAD)" --output plan.json +uv run tools/inference_service.py launch --plan plan.json --output launch.json +uv run tools/inference_service.py inspect --receipt launch.json +uv run tools/inference_service.py probe --plan plan.json +uv run tools/inference_service.py cancel --receipt launch.json ``` -Plans serialize only that source name and render the service environment value -as ``. `launch` maps it to `VLLM_API_KEY` without -putting the value in process arguments or Docker command metadata. Docker uses -`--env VLLM_API_KEY` to inherit the resolved value. Launch fails before starting -the process or container when the source variable is absent. Do not commit local -secret files. - -## Inspect, Probe, and Cancel - -Use the plan to collect a fresh capability receipt, or the launch receipt to -inspect and stop the managed service: - -```bash -uv run tools/inference_service.py probe \ - --plan gliner-plan.json \ - --output gliner-probe.json - -uv run tools/inference_service.py inspect \ - --receipt gliner-launch.json \ - --output gliner-status.json +The `[local]` section holds host, port, and bounded startup/shutdown timeouts. +The `[vllm]` section holds tensor parallelism, memory and model limits, API-key +environment source, LoRA, eager/prefix/async controls, and Mamba controls. +Secrets remain symbolic in plans and are read from their named environment +variable only at launch. Probes use `/v1/models` plus a task-aware chat payload; +the generation probe accepts reasoning-aware GPT-OSS responses. -uv run tools/inference_service.py cancel \ - --receipt gliner-launch.json \ - --output gliner-cancellation.json -``` +## GLiNER through vLLM Factory -Local-process logs are written under `.inference-service-runs/` by default. -Use `launch --log-directory PATH` to select another location. +Factory-backed detection keeps vLLM Factory's pooling endpoint and adds the +OpenAI-compatible chat contract used by Anonymizer. The adapter preserves +dynamic labels, offsets, scores, and overlapping character chunks. Model +preparation uses the profile's pinned Hugging Face revision. ## Connect Anonymizer -The factory-backed detector, native detector, and generation server expose -OpenAI-compatible URLs. Add the compiled endpoint to a custom provider file: - -```yaml title="providers.yaml" -providers: - - name: local-inference - endpoint: http://127.0.0.1:8000/v1 - provider_type: openai - api_key: EMPTY -``` +Use the plan endpoint and served model name in a custom DataDesigner provider +and model configuration. Custom model configuration replaces Anonymizer's +bundled model pool, so retain every alias required by the roles you use. See +[Custom models](models.md#custom-models) for the role map. -Set the selected model configuration's `provider` to `local-inference` and its -`model` to the served model name. Custom `model_configs` replaces Anonymizer's -entire bundled model pool, so retain every alias required by the roles you use. -See [Custom models](models.md#custom-models) for the role map and validation -command. +Compilation proves static compatibility. The launch probe proves the observed +endpoint contract. Run Anonymizer preview and evaluation before trusting a new +model for privacy or utility. -Compilation proves only static compatibility. The launch probe proves the -observed endpoint shape. Neither proves that a model meets your privacy or -utility requirements; run Anonymizer preview and evaluation before trusting a -new model or engine combination. +Docker placement, native Transformers GLiNER serving, and cache discovery are +not supported by this tool. diff --git a/docs/concepts/models.md b/docs/concepts/models.md index 690cb1d9..42873b9c 100644 --- a/docs/concepts/models.md +++ b/docs/concepts/models.md @@ -38,7 +38,7 @@ Pass `model_providers` when you need a non-default endpoint — for example Open For managed GLiNER through vLLM Factory, the native detector fallback, and vLLM generation endpoints, see [Run local inference services](inference-services.md). -That guide covers immutable plans, local processes, Docker, capability +That guide covers immutable plans, local processes, and capability receipts, and provider configuration. Set your API keys first: diff --git a/docs/concepts/self-hosting-gliner.md b/docs/concepts/self-hosting-gliner.md index a92aea26..01363c93 100644 --- a/docs/concepts/self-hosting-gliner.md +++ b/docs/concepts/self-hosting-gliner.md @@ -86,11 +86,8 @@ subset spans before score-based deduplication across chunk overlaps. A request without `labels` returns an empty entity list so DataDesigner's generic model health check can validate the endpoint without running meaningless inference. -The native fallback still lives at -`tools/inference_service_compiler/native_gliner.py`. It has its own uv-managed -dependencies and supports `DEVICE`, `GLINER_BATCH_MODE`, -`GLINER_MAX_BATCH_REQUESTS`, and `GLINER_BATCH_WAIT_MS`. See -[Native GLiNER fallback](inference-services.md#native-gliner-fallback). +The pinned vLLM Factory profiles are the only supported local detector runtime. +Native Transformers GLiNER fallback serving is intentionally not included. --- diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index c3773a93..b7d0fd38 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Behavior tests for the source-tree inference service compiler.""" +"""Observable contracts for the local-process vLLM inference host.""" from __future__ import annotations @@ -17,729 +17,275 @@ import pytest from pydantic import ValidationError -REPO_ROOT = Path(__file__).resolve().parents[2] -CLI_PATH = REPO_ROOT / "tools" / "inference_service.py" -TOOLS_ROOT = REPO_ROOT / "tools" -NATIVE_GLINER_PATH = TOOLS_ROOT / "inference_service_compiler" / "native_gliner.py" -VLLM_SERVER_PATH = TOOLS_ROOT / "inference_service_compiler" / "vllm_server.py" -PROFILE_ROOT = TOOLS_ROOT / "inference_service_profiles" -REMOVED_GLINER_PATH = TOOLS_ROOT / "serve_gliner.py" +ROOT = Path(__file__).resolve().parents[2] +TOOLS = ROOT / "tools" +PROFILES = TOOLS / "inference_service_profiles" +CLI = TOOLS / "inference_service.py" -def load_compiler_modules() -> tuple[ModuleType, ModuleType]: - """Load source-tree modules through the same import root as the CLI.""" - sys.path.insert(0, str(TOOLS_ROOT)) +def modules() -> tuple[ModuleType, ModuleType, ModuleType]: + sys.path.insert(0, str(TOOLS)) try: - models = importlib.import_module("inference_service_compiler.models") - compiler = importlib.import_module("inference_service_compiler.compiler") - finally: - sys.path.pop(0) - return models, compiler - - -def load_cli_module() -> ModuleType: - """Load the CLI through the source-tree import root.""" - sys.path.insert(0, str(TOOLS_ROOT)) - try: - return importlib.import_module("inference_service_compiler.cli") + return ( + importlib.import_module("inference_service_compiler.models"), + importlib.import_module("inference_service_compiler.compiler"), + importlib.import_module("inference_service_compiler.runtime"), + ) finally: sys.path.pop(0) -def load_runtime_module() -> ModuleType: - """Load the runtime through the source-tree import root.""" - sys.path.insert(0, str(TOOLS_ROOT)) - try: - return importlib.import_module("inference_service_compiler.runtime") - finally: - sys.path.pop(0) +def generation(models: ModuleType, **vllm: object) -> Any: + return models.InferenceIntent( + task=models.Generation(), + model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b", revision="abc"), + vllm=models.Vllm(**vllm), + local=models.LocalProcess(port=8000, startup_timeout_seconds=3, shutdown_timeout_seconds=0.01), + ) -def build_generation_plan( - models: ModuleType, - compiler: ModuleType, - *, - api_key_env: str | None = None, - docker: bool = False, -) -> Any: - """Build one local vLLM plan without runtime effects.""" - intent = models.InferenceIntent( - task=models.Generation(chat=True), - model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b", revision="abcdef0123456789"), - engine=models.VllmEngine(api_key_env=api_key_env), - placement=( - models.DockerPlacement( - host="127.0.0.1", - port=8000, - image="vllm/vllm-openai:v0.27.1", - gpus="all", - ) - if docker - else models.LocalProcessPlacement(host="127.0.0.1", port=8000) +def launch_receipt(models: ModuleType, plan: Any, handle: Any) -> Any: + return models.LaunchReceipt( + plan_digest=plan.plan_digest, + launched_at="2026-08-07T00:00:00+00:00", + shutdown_timeout_seconds=0.01, + handle=handle, + probe=models.CapabilityProbeReceipt( + plan_digest=plan.plan_digest, + endpoint=plan.endpoint, + observed_at="2026-08-07T00:00:00+00:00", + models=(plan.expected_model,), + observed_capabilities=plan.required_capabilities, + passed=True, ), - access=models.DirectAccess(), - lifecycle=models.ManagedLifecycle(startup_timeout_seconds=30), ) - return compiler.compile_intent(intent, source_revision="3f68c145") - - -def test_cli_is_a_directly_executable_source_tree_entrypoint() -> None: - """The compiler has one stable CLI without creating an installable package.""" - assert CLI_PATH.is_file() - assert CLI_PATH.stat().st_mode & stat.S_IXUSR - -def test_compile_native_gliner_local_process_plan() -> None: - """Native detection compiles into a deterministic effect-free local plan.""" - models, compiler = load_compiler_modules() - assert callable(getattr(compiler, "compile_intent", None)) - intent = models.InferenceIntent( - task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), - model=models.HuggingFaceModel(model_id="nvidia/gliner-pii", revision="0123456789abcdef"), - engine=models.NativeGlinerEngine(family="nvidia-gliner", device="cpu"), - placement=models.LocalProcessPlacement(host="127.0.0.1", port=8001), - access=models.DirectAccess(), - lifecycle=models.ManagedLifecycle(startup_timeout_seconds=120), - ) - - first = compiler.compile_intent(intent, source_revision="3f68c145") - second = compiler.compile_intent(intent, source_revision="3f68c145") - - assert first == second - assert first.schema_version == "inference-service.run-plan/v1" - assert first.intent_digest == compiler.digest_model(intent) - assert first.plan_digest == compiler.digest_plan(first) - assert first.endpoint.url == "http://127.0.0.1:8001/v1" - assert first.readiness.url == "http://127.0.0.1:8001/v1/models" - assert first.expected_model == "nvidia/gliner-pii" - assert first.declared_capabilities == ("dynamic-labels", "offsets", "scores") - assert first.required_capabilities == ("dynamic-labels", "offsets", "scores") - assert first.runtime.kind == "local-process" - assert first.command.render_argv() == ( - "uv", - "run", - "--script", - "tools/inference_service_compiler/native_gliner.py", - "--host", - "127.0.0.1", - "--port", - "8001", - "--model", - "nvidia-gliner", - "--checkpoint", - "nvidia/gliner-pii", - "--revision", - "0123456789abcdef", - ) +def test_cli_remains_a_directly_executable_source_entrypoint() -> None: + assert CLI.is_file() + assert CLI.stat().st_mode & stat.S_IXUSR - with pytest.raises(Exception, match="frozen"): - first.endpoint.port = 9000 +def test_all_shipped_profiles_compile() -> None: + models, compiler, _runtime = modules() + load_profile = importlib.import_module("inference_service_compiler.profiles").load_profile -def test_compile_vllm_docker_plan_keeps_secrets_symbolic() -> None: - """Docker plans pin the image and never serialize an API-key value.""" - models, compiler = load_compiler_modules() - intent = models.InferenceIntent( - task=models.Generation(chat=True), - model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b", revision="abcdef0123456789"), - engine=models.VllmEngine( - served_model_name="anonymizer-local", - api_key_env="LOCAL_VLLM_API_KEY", - tensor_parallel_size=2, - gpu_memory_utilization=0.8, - max_model_len=4096, - eager=True, - ), - placement=models.DockerPlacement( - host="127.0.0.1", - port=8000, - image="vllm/vllm-openai:v0.27.1", - gpus="all", - ), - access=models.DirectAccess(), - lifecycle=models.ManagedLifecycle(startup_timeout_seconds=300), - ) + plans = [ + compiler.compile_intent(load_profile(path), source_revision="test") for path in sorted(PROFILES.glob("*.toml")) + ] + assert len(plans) == 7 + assert all(plan.schema_version == "inference-service.run-plan/v2" for plan in plans) - plan = compiler.compile_intent(intent, source_revision="3f68c145") - rendered = plan.model_dump_json() - assert plan.runtime.kind == "docker" - assert plan.runtime.image == "vllm/vllm-openai:v0.27.1" - assert plan.endpoint.url == "http://127.0.0.1:8000/v1" +def test_compile_command_writes_a_digest_verified_plan(tmp_path: Path) -> None: + _models, compiler, _runtime = modules() + cli = importlib.import_module("inference_service_compiler.cli") + output = tmp_path / "plan.json" + with pytest.raises(SystemExit) as exc_info: + cli.app( + [ + "compile", + "--profile", + str(PROFILES / "vllm-local.toml"), + "--source-revision", + "test", + "--output", + str(output), + ] + ) + assert exc_info.value.code == 0 + plan = compiler.load_plan(output.read_text(encoding="utf-8")) assert plan.expected_model == "anonymizer-local" - assert plan.required_capabilities == ("chat-completions",) - assert plan.declared_capabilities == ("chat-completions",) - assert "LOCAL_VLLM_API_KEY" in rendered - assert "test-secret" not in rendered - assert "test-secret" not in plan.command.render_argv() - assert plan.command.render_environment() == {"VLLM_API_KEY": ""} - assert plan.command.render_environment(resolve_secrets={"LOCAL_VLLM_API_KEY": "test-secret"}) == { - "VLLM_API_KEY": "test-secret" - } - assert plan.command.render_argv()[:6] == ( - "docker", - "run", - "--detach", - "--rm", - "--gpus", - "all", - ) - environment_index = plan.command.render_argv().index("--env") - assert plan.command.render_argv()[environment_index : environment_index + 2] == ("--env", "VLLM_API_KEY") - revision_index = plan.command.render_argv().index("--revision") - assert plan.command.render_argv()[revision_index : revision_index + 4] == ( - "--revision", - "abcdef0123456789", - "--tokenizer-revision", - "abcdef0123456789", - ) - - -def test_compile_local_vllm_plan_uses_the_python_server_factory() -> None: - """Local vLLM runs through the source-owned Python runtime, not its CLI binary.""" - models, compiler = load_compiler_modules() - intent = models.InferenceIntent( - task=models.Generation(chat=True), - model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b", revision="abcdef0123456789"), - engine=models.VllmEngine(python_executable=".venv/bin/python"), - placement=models.LocalProcessPlacement(host="127.0.0.1", port=8000), - access=models.DirectAccess(), - lifecycle=models.ManagedLifecycle(), - ) - plan = compiler.compile_intent(intent, source_revision="3f68c145") - assert plan.command.render_argv()[:3] == ( - ".venv/bin/python", - "tools/inference_service_compiler/vllm_server.py", - "openai/gpt-oss-20b", +def test_generation_argv_keeps_local_vllm_controls_and_omits_defaults() -> None: + models, compiler, _runtime = modules() + plan = compiler.compile_intent( + generation(models, api_key_env="LOCAL_KEY", tensor_parallel_size=2, max_model_len=4096, eager=True), + source_revision="test", ) - assert "serve" not in plan.command.render_argv() - assert VLLM_SERVER_PATH.is_file() - - -def test_compiler_accepts_gliner_through_external_vllm_factory() -> None: - """GLiNER compiles through the pinned factory plugin and Python vLLM runtime.""" - models, compiler = load_compiler_modules() - intent = models.InferenceIntent( - task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), - model=models.HuggingFaceModel(model_id="nvidia/gliner-pii", revision="bd23e8ef4425fd04"), - engine=models.VllmEngine(factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), - placement=models.LocalProcessPlacement(), - access=models.DirectAccess(), - lifecycle=models.ManagedLifecycle(), - ) - - plan = compiler.compile_intent(intent, source_revision="3f68c145") - - assert plan.declared_capabilities == ("dynamic-labels", "offsets", "scores") - assert "vllm==0.27.1" in plan.dependencies - assert ( - "vllm-factory[gliner] @ git+https://github.com/latenceainew/vllm-factory.git@" - "7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0" - ) in plan.dependencies - assert "--vllm-factory-plugin" in plan.command.render_argv() - assert "deberta_gliner" in plan.command.render_argv() - - -def test_compiler_rejects_gliner_through_stock_vllm() -> None: - """Entity detection requires an explicit vLLM Factory plugin.""" - models, compiler = load_compiler_modules() - intent = models.InferenceIntent( + argv = plan.command.render_argv() + assert argv[:3] == (".venv/bin/python", "tools/inference_service_compiler/vllm_server.py", "openai/gpt-oss-20b") + assert ("--tensor-parallel-size", "2") == argv[ + argv.index("--tensor-parallel-size") : argv.index("--tensor-parallel-size") + 2 + ] + assert "--enforce-eager" in argv and "--enable-prefix-caching" not in argv + assert plan.command.render_environment() == {"VLLM_API_KEY": ""} + + +def test_factory_detection_is_task_bounded() -> None: + models, compiler, _runtime = modules() + valid = models.InferenceIntent( task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), - model=models.HuggingFaceModel(model_id="nvidia/gliner-pii"), - engine=models.VllmEngine(), - placement=models.LocalProcessPlacement(), - access=models.DirectAccess(), - lifecycle=models.ManagedLifecycle(), + model=models.HuggingFaceModel(model_id="nvidia/gliner-pii", revision="abc"), + vllm=models.Vllm(factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), + local=models.LocalProcess(), ) - - with pytest.raises(compiler.CompilationError) as exc_info: - compiler.compile_intent(intent, source_revision="3f68c145") - - assert exc_info.value.diagnostic.code == "unsupported-task-engine" - - -def test_compiler_rejects_unpinned_or_uncharacterized_factory_models() -> None: - """Factory plans close both checkpoint provenance and plugin compatibility.""" - models, compiler = load_compiler_modules() - - def compile_model(model_id: str, revision: str | None): - return compiler.compile_intent( - models.InferenceIntent( - task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), - model=models.HuggingFaceModel(model_id=model_id, revision=revision), - engine=models.VllmEngine(factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), - placement=models.LocalProcessPlacement(), - access=models.DirectAccess(), - lifecycle=models.ManagedLifecycle(), - ), - source_revision="3f68c145", + assert "--vllm-factory-plugin" in compiler.compile_intent(valid, source_revision="test").command.render_argv() + with pytest.raises(compiler.CompilationError, match="does not support"): + compiler.compile_intent( + generation(models, factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), source_revision="test" ) - with pytest.raises(compiler.CompilationError) as unpinned: - compile_model("nvidia/gliner-pii", None) - assert unpinned.value.diagnostic.code == "unpinned-model-revision" - - with pytest.raises(compiler.CompilationError) as unsupported: - compile_model("urchade/gliner_small-v2.1", "abcdef0123456789") - assert unsupported.value.diagnostic.code == "unsupported-model-engine" - - -def test_compiler_rejects_unsupported_native_generation() -> None: - """Compatibility failures are typed compiler diagnostics, not runtime surprises.""" - models, compiler = load_compiler_modules() - intent = models.InferenceIntent( - task=models.Generation(chat=True), - model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b"), - engine=models.NativeGlinerEngine(), - placement=models.LocalProcessPlacement(), - access=models.DirectAccess(), - lifecycle=models.ManagedLifecycle(), - ) - - with pytest.raises(compiler.CompilationError) as exc_info: - compiler.compile_intent(intent, source_revision="3f68c145") - - assert exc_info.value.diagnostic.code == "unsupported-task-engine" - assert exc_info.value.diagnostic.details == { - "engine": "native-gliner", - "task": "generation", - } - -def test_transport_rejects_unknown_fields_and_attach_variants() -> None: - """The initial union is closed and has no attach or external-endpoint path.""" - models, _compiler = load_compiler_modules() - payload = { - "schema_version": "inference-service.intent/v1", - "task": {"kind": "generation", "chat": True}, - "model": {"kind": "hugging-face", "model_id": "openai/gpt-oss-20b"}, - "engine": {"kind": "vllm"}, - "placement": {"kind": "attach", "url": "http://example.invalid/v1"}, - "access": {"kind": "direct"}, - "lifecycle": {"kind": "managed"}, - "unexpected": True, - } +def test_factory_detection_requires_a_pin_and_characterized_model() -> None: + models, compiler, _runtime = modules() + for model_id, revision, message in ( + ("nvidia/gliner-pii", None, "pinned model revision"), + ("unknown/model", "abc", "not characterized"), + ): + intent = models.InferenceIntent( + task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), + model=models.HuggingFaceModel(model_id=model_id, revision=revision), + vllm=models.Vllm(factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), + local=models.LocalProcess(), + ) + with pytest.raises(compiler.CompilationError, match=message): + compiler.compile_intent(intent, source_revision="test") - with pytest.raises(Exception): - models.InferenceIntent.model_validate(payload) - with pytest.raises(Exception): - models.Generation(chat=False) +def test_removed_domains_are_invalid_profile_fields() -> None: + models, _compiler, _runtime = modules() + with pytest.raises(ValidationError): + models.InferenceIntent.model_validate( + { + "schema_version": "inference-service.intent/v2", + "task": {"kind": "generation"}, + "model": {"model_id": "x"}, + "vllm": {}, + "local": {}, + "placement": {"kind": "docker"}, + } + ) def test_plan_digest_detects_transport_mutation() -> None: - """Loading a modified plan fails before any runtime effect can occur.""" - models, compiler = load_compiler_modules() - intent = models.InferenceIntent( - task=models.Generation(chat=True), - model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b"), - engine=models.VllmEngine(), - placement=models.LocalProcessPlacement(), - access=models.DirectAccess(), - lifecycle=models.ManagedLifecycle(), - ) - plan = compiler.compile_intent(intent, source_revision="3f68c145") - payload = json.loads(plan.model_dump_json()) - payload["endpoint"]["port"] = 9000 - + models, compiler, _runtime = modules() + plan = compiler.compile_intent(generation(models), source_revision="test") + changed = json.loads(plan.model_dump_json()) + changed["endpoint"]["port"] = 9000 with pytest.raises(compiler.PlanIntegrityError, match="plan digest mismatch"): - compiler.load_plan(json.dumps(payload)) - + compiler.load_plan(json.dumps(changed)) -def test_compile_command_accepts_toml_profile_and_writes_json_plan(tmp_path: Path) -> None: - """Operators author TOML while generated plans retain the JSON transport.""" - cli = load_cli_module() - profile_path = tmp_path / "generation.toml" - plan_path = tmp_path / "plan.json" - profile_path.write_text( - """\ -schema_version = "inference-service.intent/v1" -[task] -kind = "generation" -chat = true - -[model] -kind = "hugging-face" -model_id = "openai/gpt-oss-20b" - -[engine] -kind = "vllm" - -[placement] -kind = "local-process" -host = "127.0.0.1" -port = 8000 - -[access] -kind = "direct" - -[lifecycle] -kind = "managed" -""", - encoding="utf-8", +def test_lora_is_rendered_as_a_model_artifact() -> None: + models, compiler, _runtime = modules() + intent = models.InferenceIntent( + task=models.Generation(), + model=models.HuggingFaceModel( + model_id="openai/gpt-oss-20b", + adapter=models.LoraAdapter(path="/models/privacy-adapter", name="privacy"), + ), + vllm=models.Vllm(), + local=models.LocalProcess(), ) + argv = compiler.compile_intent(intent, source_revision="test").command.render_argv() + assert argv[-3:] == ("--enable-lora", "--lora-modules", "privacy=/models/privacy-adapter") - with mock.patch.object(sys, "argv", ["inference-service"]): - with pytest.raises(SystemExit) as exc_info: - cli.app( - [ - "compile", - "--profile", - str(profile_path), - "--source-revision", - "3f68c145", - "--output", - str(plan_path), - ] - ) - - assert exc_info.value.code == 0 - payload = json.loads(plan_path.read_text(encoding="utf-8")) - assert payload["schema_version"] == "inference-service.run-plan/v1" - assert payload["source_revision"] == "3f68c145" - assert payload["plan_digest"] - - -def test_equivalent_toml_profiles_compile_to_the_same_plan(tmp_path: Path) -> None: - """TOML comments and table order do not change semantic plan identity.""" - cli = load_cli_module() - _models, compiler = load_compiler_modules() - first_path = tmp_path / "first.toml" - second_path = tmp_path / "second.toml" - first_path.write_text( - """\ -schema_version = "inference-service.intent/v1" -[task] -kind = "generation" -chat = true -[model] -kind = "hugging-face" -model_id = "openai/gpt-oss-20b" -[engine] -kind = "vllm" -[placement] -kind = "local-process" -host = "127.0.0.1" -port = 8000 -[access] -kind = "direct" -[lifecycle] -kind = "managed" -""", - encoding="utf-8", - ) - second_path.write_text( - """\ -# The order and formatting are for humans; the compiler sees one typed value. -schema_version = "inference-service.intent/v1" -[lifecycle] -kind = "managed" -[access] -kind = "direct" -[placement] -port = 8000 -host = "127.0.0.1" -kind = "local-process" -[engine] -kind = "vllm" -[model] -model_id = "openai/gpt-oss-20b" -kind = "hugging-face" -[task] -chat = true -kind = "generation" -""", - encoding="utf-8", - ) - first = compiler.compile_intent(cli.load_profile(first_path), source_revision="3f68c145") - second = compiler.compile_intent(cli.load_profile(second_path), source_revision="3f68c145") - - assert first == second - - -def test_toml_profile_rejects_unknown_engine_fields(tmp_path: Path) -> None: - """Closed profile tables reject unknown settings before compilation.""" - cli = load_cli_module() - profile_path = tmp_path / "invalid.toml" - profile_path.write_text( - """\ -schema_version = "inference-service.intent/v1" -[task] -kind = "generation" -chat = true -[model] -kind = "hugging-face" -model_id = "openai/gpt-oss-20b" -[engine] -kind = "vllm" -unknown = true -[placement] -kind = "local-process" -[access] -kind = "direct" -[lifecycle] -kind = "managed" -""", - encoding="utf-8", - ) +def test_probe_payload_is_task_aware_and_reasoning_safe() -> None: + models, compiler, runtime = modules() + plan = compiler.compile_intent(generation(models), source_revision="test") + requests: list[httpx.Request] = [] - with pytest.raises(ValidationError): - cli.load_profile(profile_path) - - -def test_reference_toml_profiles_are_pinned_and_compile() -> None: - """Bundled operator profiles stay parseable, pinned, and compatible.""" - cli = load_cli_module() - _models, compiler = load_compiler_modules() - - profile_paths = tuple(sorted(PROFILE_ROOT.glob("*.toml"))) - - assert {path.name for path in profile_paths} == { - "gliner2.toml", - "gpt-oss-120b.toml", - "gpt-oss-20b.toml", - "nemotron-3.5-lightning.toml", - "nvidia-gliner.toml", - "qwen3-30b-a3b-instruct.toml", - "vllm-local.toml", - } - for path in profile_paths: - intent = cli.load_profile(path) - assert intent.model.revision is not None - plan = compiler.compile_intent(intent, source_revision="3f68c145") - assert plan - if path.name == "nemotron-3.5-lightning.toml": - assert plan.dependencies == ( - "vllm==0.27.1", - "nvidia-cuda-nvcc==13.0.88", - "nvidia-cuda-crt==13.0.88", - "nvidia-nvvm==13.0.88", - ) - - -def test_probe_records_generation_capabilities() -> None: - """A live probe records models and observed task capabilities in a v1 receipt.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - assert callable(getattr(runtime, "probe_endpoint", None)) - plan = build_generation_plan(models, compiler) - - def respond(request: httpx.Request) -> httpx.Response: + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) if request.url.path == "/v1/models": return httpx.Response(200, json={"data": [{"id": "openai/gpt-oss-20b"}]}) - if request.url.path == "/v1/chat/completions": - return httpx.Response(200, json={"choices": [{"message": {"content": "ready"}}]}) - return httpx.Response(404) + return httpx.Response(200, json={"choices": [{"message": {"content": "ok", "reasoning_content": "r"}}]}) - with httpx.Client(transport=httpx.MockTransport(respond)) as client: - receipt = runtime.probe_endpoint(plan, client=client) - - assert receipt.schema_version == "inference-service.capability-probe-receipt/v1" - assert receipt.plan_digest == plan.plan_digest - assert receipt.models == ("openai/gpt-oss-20b",) - assert receipt.observed_capabilities == ("chat-completions",) - assert receipt.passed is True - - -def test_probe_supports_reasoning_models() -> None: - """The capability probe obtains content from models that reason before answering.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - plan = build_generation_plan(models, compiler) - - def respond(request: httpx.Request) -> httpx.Response: - if request.url.path == "/v1/models": - return httpx.Response(200, json={"data": [{"id": "openai/gpt-oss-20b"}]}) - if request.url.path == "/v1/chat/completions": - payload = json.loads(request.content) - enough_output = payload.get("max_tokens", 0) > 8 - low_reasoning = payload.get("chat_template_kwargs", {}).get("reasoning_effort") == "low" - content = "ready" if enough_output and low_reasoning else None - return httpx.Response(200, json={"choices": [{"message": {"content": content}}]}) - return httpx.Response(404) - - with httpx.Client(transport=httpx.MockTransport(respond)) as client: + with httpx.Client(transport=httpx.MockTransport(handler)) as client: receipt = runtime.probe_endpoint(plan, client=client) - - assert receipt.passed is True + assert receipt.passed and receipt.observed_capabilities == ("chat-completions",) + payload = json.loads(requests[-1].content) + assert payload["max_tokens"] == 128 + assert payload["chat_template_kwargs"] == {"enable_thinking": False, "reasoning_effort": "low"} -def test_probe_uses_the_resolved_bearer_secret() -> None: - """Secured readiness and task probes authenticate without serializing the value.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - plan = build_generation_plan(models, compiler, api_key_env="LOCAL_VLLM_API_KEY") +def test_probe_uses_bearer_secret_without_serializing_it() -> None: + models, compiler, runtime = modules() + plan = compiler.compile_intent(generation(models, api_key_env="LOCAL_KEY"), source_revision="test") - def respond(request: httpx.Request) -> httpx.Response: + def handler(request: httpx.Request) -> httpx.Response: assert request.headers["authorization"] == "Bearer test-secret" if request.url.path == "/v1/models": - return httpx.Response(200, json={"data": [{"id": "openai/gpt-oss-20b"}]}) - if request.url.path == "/v1/chat/completions": - return httpx.Response(200, json={"choices": [{"message": {"content": "ready"}}]}) - return httpx.Response(404) - - with httpx.Client(transport=httpx.MockTransport(respond)) as client: - receipt = runtime.probe_endpoint( - plan, - client=client, - secret_values={"LOCAL_VLLM_API_KEY": "test-secret"}, - ) + return httpx.Response(200, json={"data": [{"id": plan.expected_model}]}) + return httpx.Response(200, json={"choices": [{"message": {"content": "ready"}}]}) - assert receipt.passed is True + with httpx.Client(transport=httpx.MockTransport(handler)) as client: + receipt = runtime.probe_endpoint(plan, client=client, secret_values={"LOCAL_KEY": "test-secret"}) + assert receipt.passed assert "test-secret" not in receipt.model_dump_json() -def test_probe_rejects_the_wrong_served_model() -> None: - """Capabilities from another model do not satisfy the compiled contract.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - plan = build_generation_plan(models, compiler) +def test_probe_rejects_wrong_model_and_status() -> None: + models, compiler, runtime = modules() + plan = compiler.compile_intent(generation(models), source_revision="test") - def respond(request: httpx.Request) -> httpx.Response: + def wrong_model(request: httpx.Request) -> httpx.Response: if request.url.path == "/v1/models": return httpx.Response(200, json={"data": [{"id": "other/model"}]}) - if request.url.path == "/v1/chat/completions": - return httpx.Response(200, json={"choices": [{"message": {"content": "ready"}}]}) - return httpx.Response(404) - - with httpx.Client(transport=httpx.MockTransport(respond)) as client: - receipt = runtime.probe_endpoint(plan, client=client) + return httpx.Response(200, json={"choices": [{"message": {"content": "ready"}}]}) - assert receipt.models == ("other/model",) - assert receipt.observed_capabilities == ("chat-completions",) - assert receipt.passed is False - - -def test_probe_enforces_the_declared_readiness_status() -> None: - """A parseable response with the wrong status does not satisfy readiness.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - plan = build_generation_plan(models, compiler) + with httpx.Client(transport=httpx.MockTransport(wrong_model)) as client: + assert runtime.probe_endpoint(plan, client=client).passed is False + with httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(201, json={}))) as client: + with pytest.raises(runtime.RuntimeEffectError, match="status 201"): + runtime.probe_endpoint(plan, client=client) - def respond(request: httpx.Request) -> httpx.Response: - return httpx.Response(201, json={"data": [{"id": "openai/gpt-oss-20b"}]}) - with httpx.Client(transport=httpx.MockTransport(respond)) as client: - with pytest.raises(runtime.RuntimeEffectError, match="readiness probe returned status 201, expected 200"): - runtime.probe_endpoint(plan, client=client) +def test_plan_integrity_and_pid_cleanup_are_enforced(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + models, compiler, runtime = modules() + plan = compiler.compile_intent(generation(models), source_revision="test") + with pytest.raises(compiler.PlanIntegrityError): + runtime.launch_plan( + plan.model_copy(update={"source_revision": "changed"}), secret_values={}, log_directory=tmp_path + ) + handle = models.LocalProcessHandle( + external_id="1:x", pid=1, process_group_id=1, start_marker="old", stdout_path="out", stderr_path="err" + ) + monkeypatch.setattr(runtime, "read_process_start_marker", lambda _pid: "new") + assert runtime.is_handle_running(handle) is False -def test_launch_local_process_returns_reconnectable_handle(tmp_path: Path) -> None: - """Launching a plan records external process identity and readiness evidence.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - assert callable(getattr(runtime, "launch_plan", None)) - plan = build_generation_plan(models, compiler, api_key_env="LOCAL_VLLM_API_KEY") +def test_launch_records_process_identity_and_resolves_secrets(tmp_path: Path) -> None: + models, compiler, runtime = modules() + plan = compiler.compile_intent(generation(models, api_key_env="LOCAL_KEY"), source_revision="test") probe = models.CapabilityProbeReceipt( plan_digest=plan.plan_digest, endpoint=plan.endpoint, observed_at="2026-08-07T00:00:00+00:00", - models=("openai/gpt-oss-20b",), + models=(plan.expected_model,), observed_capabilities=("chat-completions",), passed=True, ) process = mock.Mock(pid=4242) - with ( mock.patch.object(runtime.subprocess, "Popen", return_value=process) as popen, - mock.patch.object(runtime, "probe_endpoint", return_value=probe), + mock.patch.object(runtime, "wait_for_readiness", return_value=probe), mock.patch.object(runtime, "read_process_start_marker", return_value="100"), - mock.patch.object(runtime, "is_handle_running", return_value=True), ): - receipt = runtime.launch_plan( - plan, - secret_values={"LOCAL_VLLM_API_KEY": "test-secret"}, - log_directory=tmp_path, - ) - - launched_argv = tuple(popen.call_args.args[0]) - assert launched_argv == plan.command.render_argv() - assert "test-secret" not in launched_argv + receipt = runtime.launch_plan(plan, secret_values={"LOCAL_KEY": "test-secret"}, log_directory=tmp_path) + assert tuple(popen.call_args.args[0]) == plan.command.render_argv() assert popen.call_args.kwargs["env"]["VLLM_API_KEY"] == "test-secret" - assert receipt.schema_version == "inference-service.launch-receipt/v1" - assert receipt.plan_digest == plan.plan_digest - assert receipt.handle.kind == "local-process" + assert "test-secret" not in popen.call_args.args[0] assert receipt.handle.external_id == "4242:100" - assert receipt.handle.pid == 4242 - assert receipt.probe == probe - assert Path(receipt.handle.stdout_path).parent == tmp_path - -def test_launch_rejects_an_unresolved_secret_before_effects(tmp_path: Path) -> None: - """Missing secret references fail before a process or container is started.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - plan = build_generation_plan(models, compiler, api_key_env="LOCAL_VLLM_API_KEY") +def test_missing_secret_fails_before_process_start(tmp_path: Path) -> None: + models, compiler, runtime = modules() + plan = compiler.compile_intent(generation(models, api_key_env="LOCAL_KEY"), source_revision="test") with mock.patch.object(runtime.subprocess, "Popen") as popen: with pytest.raises(runtime.RuntimeEffectError) as exc_info: runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) - popen.assert_not_called() assert exc_info.value.diagnostic.code == "missing-secret" assert exc_info.value.diagnostic.known_effects == () -def test_launch_rejects_an_in_memory_mutated_plan_before_effects(tmp_path: Path) -> None: - """The Python runtime boundary verifies plans as strictly as the JSON CLI.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - plan = build_generation_plan(models, compiler) - changed = plan.model_copy(update={"expected_model": "other/model"}) - - with mock.patch.object(runtime.subprocess, "Popen") as popen: - with pytest.raises(compiler.PlanIntegrityError, match="plan digest mismatch"): - runtime.launch_plan(changed, secret_values={}, log_directory=tmp_path) - - popen.assert_not_called() - - -def test_launch_docker_returns_container_identity(tmp_path: Path) -> None: - """Docker launch captures the stable container ID instead of the client PID.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - plan = build_generation_plan(models, compiler, docker=True) - probe = models.CapabilityProbeReceipt( - plan_digest=plan.plan_digest, - endpoint=plan.endpoint, - observed_at="2026-08-07T00:00:00+00:00", - models=("openai/gpt-oss-20b",), - observed_capabilities=("chat-completions",), - passed=True, - ) - completed = mock.Mock(returncode=0, stdout="abc123\n", stderr="") - - with ( - mock.patch.object(runtime.subprocess, "run", return_value=completed) as run, - mock.patch.object(runtime, "probe_endpoint", return_value=probe), - mock.patch.object(runtime, "is_handle_running", return_value=True), - ): - receipt = runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) - - assert tuple(run.call_args.args[0]) == plan.command.render_argv() - assert receipt.handle.kind == "docker" - assert receipt.handle.external_id == "abc123" - assert receipt.handle.container_id == "abc123" - - -def test_inspect_and_cancel_local_process_emit_versioned_receipts(tmp_path: Path) -> None: - """A later CLI invocation can inspect and cancel the recorded process identity.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - plan = build_generation_plan(models, compiler) - probe = models.CapabilityProbeReceipt( - plan_digest=plan.plan_digest, - endpoint=plan.endpoint, - observed_at="2026-08-07T00:00:00+00:00", - models=("openai/gpt-oss-20b",), - observed_capabilities=("chat-completions",), - passed=True, - ) +def test_inspect_cancel_and_forced_cleanup_are_versioned(tmp_path: Path) -> None: + models, compiler, runtime = modules() + plan = compiler.compile_intent(generation(models), source_revision="test") handle = models.LocalProcessHandle( external_id="4242:100", pid=4242, @@ -748,37 +294,24 @@ def test_inspect_and_cancel_local_process_emit_versioned_receipts(tmp_path: Path stdout_path=str(tmp_path / "stdout.log"), stderr_path=str(tmp_path / "stderr.log"), ) - launch = models.LaunchReceipt( - plan_digest=plan.plan_digest, - launched_at="2026-08-07T00:00:00+00:00", - shutdown_timeout_seconds=10, - handle=handle, - probe=probe, - ) - + launch = launch_receipt(models, plan, handle) with mock.patch.object(runtime, "is_handle_running", return_value=True): - status = runtime.inspect_run(launch) + assert runtime.inspect_run(launch).state == "running" with ( - mock.patch.object(runtime, "is_handle_running", side_effect=[True, False]), + mock.patch.object(runtime, "is_handle_running", side_effect=[True, True, False]), + mock.patch.object(runtime.time, "monotonic", side_effect=[0.0, 0.0, 1.0]), mock.patch.object(runtime.os, "killpg") as killpg, ): - cancellation = runtime.cancel_run(launch) - - assert status.schema_version == "inference-service.status-receipt/v1" - assert status.state == "running" - assert cancellation.schema_version == "inference-service.cancellation-receipt/v1" - assert cancellation.outcome == "terminated" - assert cancellation.cleanup_complete is True - killpg.assert_called_once_with(4242, runtime.signal.SIGTERM) + canceled = runtime.cancel_run(launch) + assert canceled.outcome == "forced" + assert canceled.cleanup_complete is True + assert [call.args[1] for call in killpg.call_args_list] == [runtime.signal.SIGTERM, runtime.signal.SIGKILL] -def test_process_identity_handles_spaces_and_zombies(tmp_path: Path) -> None: - """Linux process identity parsing handles spaced names and treats zombies as stopped.""" - models, _compiler = load_compiler_modules() - runtime = load_runtime_module() +def test_process_stat_handles_spaces_and_zombies(tmp_path: Path) -> None: + models, _compiler, runtime = modules() fields = ["S", *(str(index) for index in range(4, 22)), "98765"] - payload = f"4242 (worker with spaces) {' '.join(fields)}" - assert runtime._parse_process_stat(payload) == ("S", "98765") + assert runtime._parse_process_stat(f"4242 (worker with spaces) {' '.join(fields)}") == ("S", "98765") handle = models.LocalProcessHandle( external_id="4242:98765", pid=4242, @@ -796,106 +329,11 @@ def test_process_identity_handles_spaces_and_zombies(tmp_path: Path) -> None: kill.assert_not_called() -def test_vllm_plan_preserves_lora_model_artifact() -> None: - """LoRA remains a model artifact while vLLM owns its launch spelling.""" - models, compiler = load_compiler_modules() - intent = models.InferenceIntent( - task=models.Generation(chat=True), - model=models.HuggingFaceModel( - model_id="openai/gpt-oss-20b", - adapter=models.LoraAdapter(path="/models/privacy-adapter", name="privacy"), - ), - engine=models.VllmEngine(), - placement=models.LocalProcessPlacement(), - access=models.DirectAccess(), - lifecycle=models.ManagedLifecycle(), - ) - - plan = compiler.compile_intent(intent, source_revision="3f68c145") - - assert plan.command.render_argv()[-3:] == ( - "--enable-lora", - "--lora-modules", - "privacy=/models/privacy-adapter", - ) - - -def test_discover_cached_models_returns_versioned_source_paths(tmp_path: Path) -> None: - """Cached-model discovery survives as typed source-tree output without downloads.""" - models, _compiler = load_compiler_modules() - runtime = load_runtime_module() - assert callable(getattr(runtime, "discover_cached_models", None)) - snapshot = tmp_path / "models--openai--gpt-oss-20b" / "snapshots" / "abc123" - snapshot.mkdir(parents=True) - - result = runtime.discover_cached_models(tmp_path) - - assert result.schema_version == "inference-service.cached-models/v1" - assert result.cache_root == str(tmp_path) - assert result.models == ( - models.CachedModel(repository="openai/gpt-oss-20b", revision="abc123", snapshot_path=str(snapshot)), - ) - - -def test_models_command_writes_versioned_cache_discovery(tmp_path: Path) -> None: - """The CLI retains PR 212's cached-model discovery as typed JSON.""" - cli = load_cli_module() - cache_root = tmp_path / "hub" - snapshot = cache_root / "models--openai--gpt-oss-20b" / "snapshots" / "abc123" - snapshot.mkdir(parents=True) - output = tmp_path / "models.json" - - with pytest.raises(SystemExit) as exc_info: - cli.app(["models", "--cache-root", str(cache_root), "--output", str(output)]) - - assert exc_info.value.code == 0 - payload = json.loads(output.read_text(encoding="utf-8")) - assert payload["schema_version"] == "inference-service.cached-models/v1" - assert payload["models"][0]["repository"] == "openai/gpt-oss-20b" - - -def test_native_gliner_cutover_has_no_legacy_entrypoint() -> None: - """The characterized server moves into the compiler tree without a wrapper.""" - assert NATIVE_GLINER_PATH.is_file() - assert NATIVE_GLINER_PATH.stat().st_mode & stat.S_IXUSR - assert not REMOVED_GLINER_PATH.exists() - - -def test_native_gliner_plan_preserves_batch_environment() -> None: - """Compiler plans retain the characterized request-coalescing controls.""" - models, compiler = load_compiler_modules() - intent = models.InferenceIntent( - task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), - model=models.HuggingFaceModel(model_id="nvidia/gliner-pii"), - engine=models.NativeGlinerEngine( - device="cuda", - batch_mode=True, - max_batch_requests=64, - batch_wait_ms=10, - ), - placement=models.LocalProcessPlacement(port=9000), - access=models.DirectAccess(), - lifecycle=models.ManagedLifecycle(), - ) - - plan = compiler.compile_intent(intent, source_revision="3f68c145") - - assert {item.name: item.value for item in plan.command.environment} == { - "DEVICE": "cuda", - "GLINER_BATCH_MODE": "true", - "GLINER_MAX_BATCH_REQUESTS": "64", - "GLINER_BATCH_WAIT_MS": "10.0", - } - - -def test_failed_readiness_cleans_up_the_launched_process(tmp_path: Path) -> None: - """A failed readiness probe reports and cleans every known launch effect.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - plan = build_generation_plan(models, compiler) +def test_failed_readiness_cleans_up_the_known_process(tmp_path: Path) -> None: + models, compiler, runtime = modules() + plan = compiler.compile_intent(generation(models), source_revision="test") process = mock.Mock(pid=4242) failure = runtime.RuntimeEffectError(models.RuntimeDiagnostic(code="probe-failed", message="not ready")) - with ( mock.patch.object(runtime.subprocess, "Popen", return_value=process), mock.patch.object(runtime, "read_process_start_marker", return_value="100"), @@ -905,17 +343,14 @@ def test_failed_readiness_cleans_up_the_launched_process(tmp_path: Path) -> None ): with pytest.raises(runtime.RuntimeEffectError) as exc_info: runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) - assert exc_info.value.diagnostic.known_effects == ("4242:100",) assert exc_info.value.diagnostic.cleanup_complete is True killpg.assert_called_once_with(4242, runtime.signal.SIGTERM) -def test_readiness_stops_polling_when_the_managed_process_exits(tmp_path: Path) -> None: - """A crashed server fails immediately and points the operator to its stderr log.""" - models, compiler = load_compiler_modules() - runtime = load_runtime_module() - plan = build_generation_plan(models, compiler) +def test_readiness_stops_when_the_managed_process_exits(tmp_path: Path) -> None: + models, compiler, runtime = modules() + plan = compiler.compile_intent(generation(models), source_revision="test") handle = models.LocalProcessHandle( external_id="4242:100", pid=4242, @@ -924,14 +359,10 @@ def test_readiness_stops_polling_when_the_managed_process_exits(tmp_path: Path) stdout_path=str(tmp_path / "stdout.log"), stderr_path=str(tmp_path / "stderr.log"), ) - with ( mock.patch.object(runtime, "is_handle_running", return_value=False), mock.patch.object(runtime, "probe_endpoint") as probe, - pytest.raises(runtime.RuntimeEffectError) as exc_info, + pytest.raises(runtime.RuntimeEffectError, match="exited before readiness"), ): runtime.wait_for_readiness(plan, handle=handle) - - assert exc_info.value.diagnostic.code == "launch-exited" - assert str(tmp_path / "stderr.log") in exc_info.value.diagnostic.message probe.assert_not_called() diff --git a/tests/tools/test_native_gliner.py b/tests/tools/test_native_gliner.py deleted file mode 100644 index f8ea48c1..00000000 --- a/tests/tools/test_native_gliner.py +++ /dev/null @@ -1,284 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Behavior tests for the standalone GLiNER server without local runtimes.""" - -from __future__ import annotations - -import asyncio -import importlib.util -import inspect -import json -import stat -import sys -from pathlib import Path -from types import ModuleType -from typing import get_type_hints - -import pytest - -SCRIPT_PATH = Path(__file__).parents[2] / "tools" / "inference_service_compiler" / "native_gliner.py" - - -class FakeHTTPException(Exception): - """Small typed stand-in for FastAPI's HTTP exception.""" - - def __init__(self, *, status_code: int, detail: str) -> None: - super().__init__(detail) - self.status_code = status_code - self.detail = detail - - -def fake_module(name: str, **attributes: object) -> ModuleType: - """Build a dynamic dependency module behind one explicit boundary.""" - module = ModuleType(name) - vars(module).update(attributes) - return module - - -def load_server(monkeypatch: pytest.MonkeyPatch) -> ModuleType: - """Load the server after replacing every heavyweight or web dependency.""" - cyclopts = fake_module( - "cyclopts", - App=type("App", (), {"__init__": lambda self, **_kwargs: None, "default": lambda self, function: function}), - Parameter=object, - ) - monkeypatch.setitem(sys.modules, "cyclopts", cyclopts) - fastapi = fake_module( - "fastapi", - FastAPI=type( - "FastAPI", - (), - { - "__init__": lambda self, **_kwargs: None, - "get": lambda self, *_args, **_kwargs: lambda function: function, - "post": lambda self, *_args, **_kwargs: lambda function: function, - }, - ), - HTTPException=FakeHTTPException, - Request=object, - ) - monkeypatch.setitem(sys.modules, "fastapi", fastapi) - structlog = fake_module( - "structlog", - get_logger=lambda _name: type("Logger", (), {"info": lambda self, *_args, **_kwargs: None})(), - make_filtering_bound_logger=lambda _level: object, - configure=lambda **_kwargs: None, - dev=type("Dev", (), {"ConsoleRenderer": lambda: object()}), - processors=type("Processors", (), {"JSONRenderer": lambda: object()}), - ) - monkeypatch.setitem(sys.modules, "structlog", structlog) - monkeypatch.setitem(sys.modules, "uvicorn", ModuleType("uvicorn")) - spec = importlib.util.spec_from_file_location("native_gliner_under_test", SCRIPT_PATH) - if spec is None or spec.loader is None: - raise RuntimeError("could not create the server module specification") - module = importlib.util.module_from_spec(spec) - monkeypatch.setitem(sys.modules, spec.name, module) - spec.loader.exec_module(module) - return module - - -def test_default_nvidia_gliner_selection(monkeypatch: pytest.MonkeyPatch) -> None: - """The default family remains NVIDIA's original PII checkpoint.""" - server = load_server(monkeypatch) - config = server.ServerConfig() - assert config.model is server.ModelFamily.NVIDIA_GLINER - assert config.resolved_checkpoint == "nvidia/gliner-pii" - - -def test_gliner2_selection_and_checkpoint_override(monkeypatch: pytest.MonkeyPatch) -> None: - """GLiNER2 gets its PII default and either family accepts an override.""" - server = load_server(monkeypatch) - gliner2 = server.ServerConfig(model=server.ModelFamily.GLINER2) - overridden = server.ServerConfig(checkpoint="organization/custom-pii") - assert gliner2.resolved_checkpoint == "fastino/gliner2-privacy-filter-PII-multi" - assert overridden.resolved_checkpoint == "organization/custom-pii" - - -def test_invalid_model_value_is_rejected(monkeypatch: pytest.MonkeyPatch) -> None: - """The typed CLI model choice rejects values outside the closed union.""" - server = load_server(monkeypatch) - with pytest.raises(ValueError, match="not-a-model"): - server.ModelFamily("not-a-model") - - -def test_load_runtime_selects_each_local_api_and_revision(monkeypatch: pytest.MonkeyPatch) -> None: - """Runtime loading dispatches by family and resolves immutable revisions.""" - server = load_server(monkeypatch) - calls: list[tuple[str, str, str, str | None]] = [] - - class NvidiaModel: - @classmethod - def from_pretrained(cls, checkpoint: str, map_location: str, revision: str | None = None) -> object: - calls.append(("nvidia", checkpoint, map_location, revision)) - return object() - - class Gliner2Model: - @classmethod - def from_pretrained(cls, checkpoint: str, map_location: str) -> object: - calls.append(("gliner2", checkpoint, map_location, None)) - return object() - - modules = { - "gliner": type("GlinerModule", (), {"GLiNER": NvidiaModel}), - "gliner2": type("Gliner2Module", (), {"GLiNER2": Gliner2Model}), - "huggingface_hub": type( - "HubModule", - (), - {"snapshot_download": staticmethod(lambda *, repo_id, revision: f"/cache/{repo_id}/{revision}")}, - ), - } - monkeypatch.setattr(server.importlib, "import_module", modules.__getitem__) - nvidia = server.load_runtime(server.ServerConfig(revision="nvidia-revision"), "cpu") - gliner2 = server.load_runtime( - server.ServerConfig( - model=server.ModelFamily.GLINER2, - checkpoint="fastino/custom", - revision="fastino-revision", - ), - "cuda", - ) - assert isinstance(nvidia, server.NvidiaGlinerRuntime) - assert isinstance(gliner2, server.Gliner2Runtime) - assert calls == [ - ("nvidia", "nvidia/gliner-pii", "cpu", "nvidia-revision"), - ("gliner2", "/cache/fastino/custom/fastino-revision", "cuda", None), - ] - - -def test_normalizes_nvidia_spans_and_confidence(monkeypatch: pytest.MonkeyPatch) -> None: - """Original GLiNER dictionaries preserve their span and score values.""" - server = load_server(monkeypatch) - entities = server.normalize_nvidia_output( - [[{"text": "Ada", "label": "person", "start": 3, "end": 6, "score": 0.91}]] - ) - assert entities == [[server.Entity("Ada", "person", 3, 6, 0.91)]] - - -def test_normalizes_gliner2_spans_and_confidence(monkeypatch: pytest.MonkeyPatch) -> None: - """GLiNER2 confidence and spans convert to Anonymizer's flat entity value.""" - server = load_server(monkeypatch) - entities = server.normalize_gliner2_output( - [{"entities": {"email": [{"text": "a@example.com", "start": 5, "end": 18, "confidence": 0.88}]}}] - ) - assert entities == [[server.Entity("a@example.com", "email", 5, 18, 0.88)]] - - -@pytest.mark.parametrize( - ("body", "message"), - [ - ({"labels": [42]}, "labels must be a list of strings"), - ({"flat_ner": "false"}, "flat_ner must be a boolean"), - ({"batch_size": 1.5}, "batch_size must be an integer"), - ({"batch_size": 0}, "batch_size must be >= 1"), - ({"threshold": "0.3"}, "threshold must be a number"), - ({"threshold": 1.1}, "threshold must be between 0 and 1"), - ], -) -def test_request_params_reject_implicit_coercions( - monkeypatch: pytest.MonkeyPatch, body: dict[str, object], message: str -) -> None: - """The functional request boundary rejects ambiguous JSON values.""" - server = load_server(monkeypatch) - with pytest.raises(server.RequestValidationError, match=message): - server.parse_detect_params(body) - - -def test_server_config_rejects_invalid_port_before_startup(monkeypatch: pytest.MonkeyPatch) -> None: - """Invalid ports fail before uvicorn can trigger model initialization.""" - server = load_server(monkeypatch) - with pytest.raises(ValueError, match="port must be between 1 and 65535"): - server.ServerConfig(port=70000) - - -def test_cli_parameters_are_named_options(monkeypatch: pytest.MonkeyPatch) -> None: - """Model and checkpoint selection remain discoverable named options.""" - server = load_server(monkeypatch) - parameters = inspect.signature(server.main).parameters - assert parameters["model"].kind is inspect.Parameter.KEYWORD_ONLY - assert parameters["checkpoint"].kind is inspect.Parameter.KEYWORD_ONLY - assert parameters["log_format"].kind is inspect.Parameter.KEYWORD_ONLY - - -def test_bad_cli_input_uses_craft_exit_code( - monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - """CLI validation exits 125 before starting the model server.""" - server = load_server(monkeypatch) - with pytest.raises(SystemExit) as exc_info: - server.main(port=70000) - assert exc_info.value.code == 125 - assert capsys.readouterr().err == "error: port must be between 1 and 65535\n" - - -def test_script_is_directly_executable() -> None: - """The uv shebang and executable mode form a usable entry point.""" - assert SCRIPT_PATH.stat().st_mode & stat.S_IXUSR - - -def test_fastapi_app_alias_is_preserved(monkeypatch: pytest.MonkeyPatch) -> None: - """Existing uvicorn module imports continue to resolve `app`.""" - server = load_server(monkeypatch) - assert server.app is server.api - - -def test_fastapi_route_uses_concrete_request_type(monkeypatch: pytest.MonkeyPatch) -> None: - """FastAPI resolves the endpoint parameter to its runtime Request class.""" - server = load_server(monkeypatch) - - assert get_type_hints(server.chat_completions)["request"] is object - - -def test_chat_completion_uses_anonymizer_json_string_contract(monkeypatch: pytest.MonkeyPatch) -> None: - """The OpenAI response embeds flat entities in `message.content` JSON.""" - server = load_server(monkeypatch) - - class Detector: - async def detect(self, _text: str, _params: object) -> list[object]: - return [server.Entity("Ada", "person", 0, 3, 0.9)] - - class Request: - async def json(self) -> dict[str, object]: - return {"messages": [{"role": "user", "content": "Ada"}], "labels": ["person"]} - - vars(server)["detector"] = Detector() - response = asyncio.run(server.chat_completions(Request())) - content = response["choices"][0]["message"]["content"] - assert json.loads(content) == {"entities": [{"text": "Ada", "label": "person", "start": 0, "end": 3, "score": 0.9}]} - - -def test_chat_completion_returns_422_for_invalid_params(monkeypatch: pytest.MonkeyPatch) -> None: - """Malformed client values are reported as validation errors, not 500s.""" - server = load_server(monkeypatch) - - class Detector: - async def detect(self, _text: str, _params: object) -> list[object]: - raise AssertionError("invalid requests must not reach inference") - - class Request: - async def json(self) -> dict[str, object]: - return {"messages": [], "labels": [42]} - - vars(server)["detector"] = Detector() - with pytest.raises(FakeHTTPException) as exc_info: - asyncio.run(server.chat_completions(Request())) - assert exc_info.value.status_code == 422 - assert exc_info.value.detail == "labels must be a list of strings" - - -def test_chat_completion_returns_422_for_invalid_messages(monkeypatch: pytest.MonkeyPatch) -> None: - """The message boundary rejects values outside the OpenAI list shape.""" - server = load_server(monkeypatch) - - class Detector: - async def detect(self, _text: str, _params: object) -> list[object]: - raise AssertionError("invalid requests must not reach inference") - - class Request: - async def json(self) -> dict[str, object]: - return {"messages": "bad", "labels": []} - - vars(server)["detector"] = Detector() - with pytest.raises(FakeHTTPException) as exc_info: - asyncio.run(server.chat_completions(Request())) - assert exc_info.value.status_code == 422 - assert exc_info.value.detail == "messages must be a list" diff --git a/tools/inference_service_compiler/cli.py b/tools/inference_service_compiler/cli.py index 08a15abd..4d613523 100644 --- a/tools/inference_service_compiler/cli.py +++ b/tools/inference_service_compiler/cli.py @@ -20,8 +20,6 @@ from inference_service_compiler.runtime import ( RuntimeEffectError, cancel_run, - default_cache_root, - discover_cached_models, inspect_run, launch_plan, probe_endpoint, @@ -55,7 +53,7 @@ def compile_plan( source_revision: str, output: Path | None = None, ) -> None: - """Compile a v1 TOML profile without performing runtime effects.""" + """Compile a v2 TOML profile without performing runtime effects.""" parsed = load_profile(profile) plan = compile_intent(parsed, source_revision=source_revision) write_json(plan, output) @@ -109,13 +107,6 @@ def cancel(*, receipt: Path, output: Path | None = None) -> None: write_json(cancel_run(launch_receipt), output) -@app.command -@command_errors -def models(*, cache_root: Path | None = None, output: Path | None = None) -> None: - """List existing Hugging Face cache snapshots without downloading models.""" - write_json(discover_cached_models(cache_root or default_cache_root()), output) - - def write_json(value: BaseModel, output: Path | None) -> None: """Write one versioned transport value to a file or standard output.""" rendered = value.model_dump_json(indent=2) + "\n" diff --git a/tools/inference_service_compiler/compiler.py b/tools/inference_service_compiler/compiler.py index 6c9e1bef..11e86976 100644 --- a/tools/inference_service_compiler/compiler.py +++ b/tools/inference_service_compiler/compiler.py @@ -16,22 +16,17 @@ CommandArgument, CommandSpec, CompatibilityEvidence, - DockerPlacement, - DockerRuntime, EndpointContract, EntityDetection, - EnvironmentVariable, FrozenModel, Generation, HttpProbe, InferenceIntent, LiteralArgument, - LocalProcessPlacement, LocalProcessRuntime, - NativeGlinerEngine, RunPlan, SecretEnvironmentVariable, - VllmEngine, + Vllm, ) from inference_service_compiler.vllm_factory_integration import ( VLLM_FACTORY_DEPENDENCY, @@ -73,7 +68,7 @@ def compile_intent(intent: InferenceIntent, *, source_revision: str) -> RunPlan: raise ValueError("source_revision must not be empty") required = intent.task.required_capabilities() command, runtime, declared, evidence = _compile_service(intent) - placement = intent.placement + placement = intent.local endpoint = EndpointContract(host=placement.host, port=placement.port) plan = RunPlan( plan_digest="", @@ -86,10 +81,8 @@ def compile_intent(intent: InferenceIntent, *, source_revision: str) -> RunPlan: host=placement.host, port=placement.port, path="/v1/models", - timeout_seconds=intent.lifecycle.startup_timeout_seconds, - bearer_token_environment_variable=( - intent.engine.api_key_env if isinstance(intent.engine, VllmEngine) else None - ), + timeout_seconds=intent.local.startup_timeout_seconds, + bearer_token_environment_variable=intent.vllm.api_key_env, ), expected_model=intent.expected_model, required_capabilities=required, @@ -114,7 +107,7 @@ def digest_plan(plan: RunPlan) -> str: def load_plan(serialized: str | bytes) -> RunPlan: - """Parse a closed v1 plan and reject transport mutation.""" + """Parse a closed v2 plan and reject transport mutation.""" plan = RunPlan.model_validate_json(serialized) verify_plan(plan) return plan @@ -133,90 +126,18 @@ def _canonical_json(value: object) -> bytes: def _compile_service( intent: InferenceIntent, -) -> tuple[ - CommandSpec, - LocalProcessRuntime | DockerRuntime, - tuple[Capability, ...], - tuple[CompatibilityEvidence, ...], -]: - match intent.engine: - case NativeGlinerEngine() as engine: - if not isinstance(intent.task, EntityDetection): - _raise_unsupported_task_engine(intent.task.kind, engine.kind) - if not isinstance(intent.placement, LocalProcessPlacement): - raise CompilationError( - CompilerDiagnostic( - code="unsupported-engine-placement", - message="native GLiNER is characterized only as a local process", - details={"engine": engine.kind, "placement": intent.placement.kind}, - ) - ) - return _compile_native_gliner(intent, engine) - case VllmEngine() as engine: - return _compile_vllm(intent, engine) - - -def _compile_native_gliner( - intent: InferenceIntent, - engine: NativeGlinerEngine, ) -> tuple[CommandSpec, LocalProcessRuntime, tuple[Capability, ...], tuple[CompatibilityEvidence, ...]]: - placement = intent.placement - if not isinstance(placement, LocalProcessPlacement): - raise TypeError(f"expected LocalProcessPlacement, got {type(placement)!r}") - argv = _literal_arguments( - "uv", - "run", - "--script", - "tools/inference_service_compiler/native_gliner.py", - "--host", - placement.host, - "--port", - str(placement.port), - "--model", - engine.family, - "--checkpoint", - intent.model.model_id, - ) - if engine.log_format != "plain": - argv += _literal_arguments("--log-format", engine.log_format) - if intent.model.revision is not None: - argv += _literal_arguments("--revision", intent.model.revision) - return ( - CommandSpec(argv=argv, environment=_native_environment(engine)), - LocalProcessRuntime(), - intent.task.required_capabilities(), - ( - CompatibilityEvidence( - rule="native-gliner-entity-detection-v1", - outcome="characterized", - detail="Anonymizer chat-completion adapter preserves dynamic labels, offsets, and scores", - ), - ), - ) - - -def _native_environment(engine: NativeGlinerEngine) -> tuple[EnvironmentVariable, ...]: - return ( - EnvironmentVariable(name="DEVICE", value=engine.device), - EnvironmentVariable(name="GLINER_BATCH_MODE", value=str(engine.batch_mode).lower()), - EnvironmentVariable(name="GLINER_MAX_BATCH_REQUESTS", value=str(engine.max_batch_requests)), - EnvironmentVariable(name="GLINER_BATCH_WAIT_MS", value=str(float(engine.batch_wait_ms))), - ) + return _compile_vllm(intent, intent.vllm) def _compile_vllm( intent: InferenceIntent, - engine: VllmEngine, -) -> tuple[ - CommandSpec, - LocalProcessRuntime | DockerRuntime, - tuple[Capability, ...], - tuple[CompatibilityEvidence, ...], -]: + engine: Vllm, +) -> tuple[CommandSpec, LocalProcessRuntime, tuple[Capability, ...], tuple[CompatibilityEvidence, ...]]: if isinstance(intent.task, EntityDetection): factory = engine.factory if factory is None: - _raise_unsupported_task_engine(intent.task.kind, engine.kind) + _raise_unsupported_task_engine(intent.task.kind, "vllm") if intent.model.revision is None: raise CompilationError( CompilerDiagnostic( @@ -244,15 +165,7 @@ def _compile_vllm( CompilerDiagnostic( code="unsupported-model-adapter", message="vLLM Factory entity detection does not support a model adapter", - details={"engine": engine.kind, "task": intent.task.kind}, - ) - ) - if isinstance(intent.placement, DockerPlacement): - raise CompilationError( - CompilerDiagnostic( - code="unsupported-engine-placement", - message="vLLM Factory entity detection is characterized only as a local process", - details={"engine": engine.kind, "placement": intent.placement.kind}, + details={"engine": "vllm", "task": intent.task.kind}, ) ) command, runtime = _vllm_command(intent, engine) @@ -272,7 +185,7 @@ def _compile_vllm( ), ) if not isinstance(intent.task, Generation) or engine.factory is not None: - _raise_unsupported_task_engine(intent.task.kind, engine.kind) + _raise_unsupported_task_engine(intent.task.kind, "vllm") command, runtime = _vllm_command(intent, engine) declared = ("chat-completions",) evidence = ( @@ -287,60 +200,23 @@ def _compile_vllm( def _vllm_command( intent: InferenceIntent, - engine: VllmEngine, -) -> tuple[CommandSpec, LocalProcessRuntime | DockerRuntime]: + engine: Vllm, +) -> tuple[CommandSpec, LocalProcessRuntime]: engine_arguments = _vllm_engine_arguments(intent, engine) - match intent.placement: - case LocalProcessPlacement() as placement: - argv = _literal_arguments( - engine.python_executable, - "tools/inference_service_compiler/vllm_server.py", - intent.model.model_id, - "--host", - placement.host, - "--port", - str(placement.port), - ) - return CommandSpec( - argv=argv + engine_arguments, environment=_vllm_environment(engine) - ), LocalProcessRuntime() - case DockerPlacement() as placement: - return _docker_vllm_command(intent, engine, placement, engine_arguments) - - -def _docker_vllm_command( - intent: InferenceIntent, - engine: VllmEngine, - placement: DockerPlacement, - engine_arguments: tuple[CommandArgument, ...], -) -> tuple[CommandSpec, DockerRuntime]: - values = [ - placement.runtime, - "run", - "--detach", - "--rm", - "--gpus", - placement.gpus, - "--ipc", - "host", - "--publish", - f"{placement.host}:{placement.port}:8000", - ] - if placement.hugging_face_cache is not None: - values.extend(["--volume", f"{placement.hugging_face_cache}:/root/.cache/huggingface"]) - if engine.api_key_env is not None: - values.extend(["--env", VLLM_API_KEY_ENV]) - values.extend([placement.image, "--model", intent.model.model_id]) - return ( - CommandSpec( - argv=_literal_arguments(*values) + engine_arguments, - environment=_vllm_environment(engine), - ), - DockerRuntime(image=placement.image), + placement = intent.local + argv = _literal_arguments( + engine.python_executable, + "tools/inference_service_compiler/vllm_server.py", + intent.model.model_id, + "--host", + placement.host, + "--port", + str(placement.port), ) + return CommandSpec(argv=argv + engine_arguments, environment=_vllm_environment(engine)), LocalProcessRuntime() -def _vllm_engine_arguments(intent: InferenceIntent, engine: VllmEngine) -> tuple[CommandArgument, ...]: +def _vllm_engine_arguments(intent: InferenceIntent, engine: Vllm) -> tuple[CommandArgument, ...]: arguments: list[CommandArgument] = [] if intent.model.revision is not None: arguments.extend( @@ -395,7 +271,7 @@ def _vllm_engine_arguments(intent: InferenceIntent, engine: VllmEngine) -> tuple return tuple(arguments) -def _vllm_environment(engine: VllmEngine) -> tuple[SecretEnvironmentVariable, ...]: +def _vllm_environment(engine: Vllm) -> tuple[SecretEnvironmentVariable, ...]: if engine.api_key_env is None: return () return ( @@ -411,14 +287,12 @@ def _literal_arguments(*values: str) -> tuple[CommandArgument, ...]: def _plan_dependencies(intent: InferenceIntent) -> tuple[str, ...]: - if isinstance(intent.engine, VllmEngine): - dependencies = [VLLM_DEPENDENCY] - if intent.engine.factory is not None: - dependencies.append(VLLM_FACTORY_DEPENDENCY) - if intent.engine.mamba_backend == "flashinfer": - dependencies.extend(FLASHINFER_CUDA_TOOLCHAIN_DEPENDENCIES) - return tuple(dependencies) - return () + dependencies = [VLLM_DEPENDENCY] + if intent.vllm.factory is not None: + dependencies.append(VLLM_FACTORY_DEPENDENCY) + if intent.vllm.mamba_backend == "flashinfer": + dependencies.extend(FLASHINFER_CUDA_TOOLCHAIN_DEPENDENCIES) + return tuple(dependencies) def _raise_unsupported_task_engine(task: str, engine: str) -> Never: diff --git a/tools/inference_service_compiler/models.py b/tools/inference_service_compiler/models.py index c855a07f..e92de96a 100644 --- a/tools/inference_service_compiler/models.py +++ b/tools/inference_service_compiler/models.py @@ -9,8 +9,8 @@ from pydantic import BaseModel, ConfigDict, Field -INTENT_SCHEMA_VERSION = "inference-service.intent/v1" -PLAN_SCHEMA_VERSION = "inference-service.run-plan/v1" +INTENT_SCHEMA_VERSION = "inference-service.intent/v2" +PLAN_SCHEMA_VERSION = "inference-service.run-plan/v2" CAPABILITY_PROBE_RECEIPT_SCHEMA_VERSION = "inference-service.capability-probe-receipt/v1" LAUNCH_RECEIPT_SCHEMA_VERSION = "inference-service.launch-receipt/v1" STATUS_RECEIPT_SCHEMA_VERSION = "inference-service.status-receipt/v1" @@ -69,27 +69,11 @@ class LoraAdapter(FrozenModel): class HuggingFaceModel(FrozenModel): """A Hugging Face model identifier and optional immutable revision.""" - kind: Literal["hugging-face"] = "hugging-face" model_id: str = Field(min_length=1) revision: str | None = Field(default=None, min_length=1) adapter: LoraAdapter | None = None -ModelSpec = Annotated[HuggingFaceModel, Field(discriminator="kind")] - - -class NativeGlinerEngine(FrozenModel): - """The characterized local GLiNER or GLiNER2 Python runtime.""" - - kind: Literal["native-gliner"] = "native-gliner" - family: Literal["nvidia-gliner", "gliner2"] = "nvidia-gliner" - device: str = Field(default="auto", min_length=1) - batch_mode: bool = True - max_batch_requests: int = Field(default=32, ge=1) - batch_wait_ms: float = Field(default=10, ge=0) - log_format: Literal["plain", "json"] = "plain" - - class VllmFactoryIntegration(FrozenModel): """A supported vLLM Factory structured-prediction plugin.""" @@ -97,10 +81,9 @@ class VllmFactoryIntegration(FrozenModel): prepared_model_root: str = Field(default="/tmp/anonymizer-vllm-factory", min_length=1) -class VllmEngine(FrozenModel): +class Vllm(FrozenModel): """vLLM's OpenAI-compatible server with bounded common options.""" - kind: Literal["vllm"] = "vllm" python_executable: str = Field(default=".venv/bin/python", min_length=1) served_model_name: str | None = Field(default=None, min_length=1) api_key_env: str | None = Field(default=None, min_length=1) @@ -118,70 +101,32 @@ class VllmEngine(FrozenModel): factory: VllmFactoryIntegration | None = None -EngineSpec = Annotated[NativeGlinerEngine | VllmEngine, Field(discriminator="kind")] - +class LocalProcess(FrozenModel): + """The only supported inference-host deployment domain.""" -class LocalProcessPlacement(FrozenModel): - """A process on the caller's host.""" - - kind: Literal["local-process"] = "local-process" host: str = Field(default="127.0.0.1", min_length=1) port: int = Field(default=8000, ge=1, le=65535) - -class DockerPlacement(FrozenModel): - """A managed local Docker container with direct host access.""" - - kind: Literal["docker"] = "docker" - host: str = Field(default="127.0.0.1", min_length=1) - port: int = Field(default=8000, ge=1, le=65535) - image: str = Field(min_length=1) - runtime: Literal["docker"] = "docker" - gpus: str = Field(default="all", min_length=1) - hugging_face_cache: str | None = Field(default=None, min_length=1) - - -PlacementSpec = Annotated[LocalProcessPlacement | DockerPlacement, Field(discriminator="kind")] - - -class DirectAccess(FrozenModel): - """A direct HTTP endpoint exposed by the managed runtime.""" - - kind: Literal["direct"] = "direct" - - -AccessSpec = Annotated[DirectAccess, Field(discriminator="kind")] - - -class ManagedLifecycle(FrozenModel): - """The compiler owns launch, inspection, cancellation, and cleanup.""" - - kind: Literal["managed"] = "managed" startup_timeout_seconds: float = Field(default=120, gt=0) shutdown_timeout_seconds: float = Field(default=30, gt=0) -LifecycleSpec = Annotated[ManagedLifecycle, Field(discriminator="kind")] - - class InferenceIntent(FrozenModel): """Complete semantic input to pure inference-service compilation.""" - schema_version: Literal["inference-service.intent/v1"] = INTENT_SCHEMA_VERSION + schema_version: Literal["inference-service.intent/v2"] = INTENT_SCHEMA_VERSION task: TaskSpec - model: ModelSpec - engine: EngineSpec - placement: PlacementSpec - access: AccessSpec - lifecycle: LifecycleSpec + model: HuggingFaceModel + vllm: Vllm + local: LocalProcess @property def expected_model(self) -> str: """Return the model ID that the compiled endpoint must serve.""" if self.model.adapter is not None: return self.model.adapter.name - if isinstance(self.engine, VllmEngine) and self.engine.served_model_name is not None: - return self.engine.served_model_name + if self.vllm.served_model_name is not None: + return self.vllm.served_model_name return self.model.model_id @@ -280,17 +225,6 @@ class LocalProcessRuntime(FrozenModel): cleanup: Literal["terminate-process-group"] = "terminate-process-group" -class DockerRuntime(FrozenModel): - """Runtime facts needed to launch and remove a local container.""" - - kind: Literal["docker"] = "docker" - image: str - cleanup: Literal["remove-container"] = "remove-container" - - -RuntimeSpec = Annotated[LocalProcessRuntime | DockerRuntime, Field(discriminator="kind")] - - class CompatibilityEvidence(FrozenModel): """One compiler rule supporting or qualifying the selected combination.""" @@ -302,12 +236,12 @@ class CompatibilityEvidence(FrozenModel): class RunPlan(FrozenModel): """Portable, immutable, effect-free instructions for one service run.""" - schema_version: Literal["inference-service.run-plan/v1"] = PLAN_SCHEMA_VERSION + schema_version: Literal["inference-service.run-plan/v2"] = PLAN_SCHEMA_VERSION plan_digest: str intent_digest: str = Field(min_length=1) intent: InferenceIntent command: CommandSpec - runtime: RuntimeSpec + runtime: LocalProcessRuntime endpoint: EndpointContract readiness: HttpProbe expected_model: str = Field(min_length=1) @@ -342,17 +276,6 @@ class LocalProcessHandle(FrozenModel): stderr_path: str -class DockerHandle(FrozenModel): - """Reconnectable identity for a managed Docker container.""" - - kind: Literal["docker"] = "docker" - external_id: str - container_id: str - - -HandleRecord = Annotated[LocalProcessHandle | DockerHandle, Field(discriminator="kind")] - - class LaunchReceipt(FrozenModel): """Known launch effects, reconnectable identity, and readiness evidence.""" @@ -360,7 +283,7 @@ class LaunchReceipt(FrozenModel): plan_digest: str launched_at: str shutdown_timeout_seconds: float = Field(gt=0) - handle: HandleRecord + handle: LocalProcessHandle probe: CapabilityProbeReceipt @@ -370,7 +293,7 @@ class StatusReceipt(FrozenModel): schema_version: Literal["inference-service.status-receipt/v1"] = STATUS_RECEIPT_SCHEMA_VERSION plan_digest: str observed_at: str - handle: HandleRecord + handle: LocalProcessHandle state: Literal["running", "stopped"] @@ -380,7 +303,7 @@ class CancellationReceipt(FrozenModel): schema_version: Literal["inference-service.cancellation-receipt/v1"] = CANCELLATION_RECEIPT_SCHEMA_VERSION plan_digest: str canceled_at: str - handle: HandleRecord + handle: LocalProcessHandle outcome: Literal["terminated", "already-stopped", "forced"] cleanup_complete: bool @@ -392,19 +315,3 @@ class RuntimeDiagnostic(FrozenModel): message: str known_effects: tuple[str, ...] = () cleanup_complete: bool | None = None - - -class CachedModel(FrozenModel): - """One immutable Hugging Face cache snapshot available to local runtimes.""" - - repository: str - revision: str - snapshot_path: str - - -class CachedModels(FrozenModel): - """Versioned discovery result that performs no model downloads.""" - - schema_version: Literal["inference-service.cached-models/v1"] = "inference-service.cached-models/v1" - cache_root: str - models: tuple[CachedModel, ...] diff --git a/tools/inference_service_compiler/native_gliner.py b/tools/inference_service_compiler/native_gliner.py deleted file mode 100755 index e3c58db2..00000000 --- a/tools/inference_service_compiler/native_gliner.py +++ /dev/null @@ -1,681 +0,0 @@ -#!/usr/bin/env -S uv run --script -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# /// script -# requires-python = ">=3.13" -# dependencies = [ -# "cyclopts>=3.0", -# "fastapi>=0.115", -# "gliner>=0.2.21", -# "gliner2[local]>=1.3", -# "structlog>=24.4", -# "uvicorn>=0.30", -# ] -# /// -"""Serve local GLiNER PII detection through Anonymizer's OpenAI wire contract. - -This server is launched by ``tools/inference_service.py`` from a compiled run -plan. The default ``nvidia-gliner`` family loads ``nvidia/gliner-pii``; -``gliner2`` loads Fastino's local PII checkpoint. -""" - -from __future__ import annotations - -import asyncio -import importlib -import json -import math -import os -import sys -import time -import uuid -from collections.abc import AsyncIterator, Mapping, Sequence -from concurrent.futures import ThreadPoolExecutor -from contextlib import asynccontextmanager -from dataclasses import dataclass -from enum import StrEnum -from typing import TYPE_CHECKING, Protocol, cast - -import structlog # ty: ignore[unresolved-import] -- optional PEP 723 server dependency -import uvicorn -from cyclopts import App - -_fastapi = importlib.import_module("fastapi") - -DEFAULT_HOST = "127.0.0.1" -DEFAULT_PORT = 8001 -DEFAULT_CHUNK_LENGTH = 384 -DEFAULT_OVERLAP = 128 -DEFAULT_FLAT_NER = False -DEFAULT_INFERENCE_BATCH_SIZE = 8 -NVIDIA_GLINER_CHECKPOINT = "nvidia/gliner-pii" -GLINER2_CHECKPOINT = "fastino/gliner2-privacy-filter-PII-multi" -BATCH_MODE = os.getenv("GLINER_BATCH_MODE", "true").lower() not in {"0", "false", "no"} -MAX_BATCH_REQUESTS = int(os.getenv("GLINER_MAX_BATCH_REQUESTS", "32")) -BATCH_WAIT_SECONDS = float(os.getenv("GLINER_BATCH_WAIT_MS", "10")) / 1000 - - -class ModelFamily(StrEnum): - """Supported local model runtime families.""" - - NVIDIA_GLINER = "nvidia-gliner" - GLINER2 = "gliner2" - - -class LogFormat(StrEnum): - """Supported server log renderers.""" - - PLAIN = "plain" - JSON = "json" - - -class RequestValidationError(ValueError): - """A malformed detector request at the pure JSON boundary.""" - - -@dataclass(frozen=True, slots=True) -class ServerConfig: - """Immutable server configuration chosen by the CLI.""" - - host: str = DEFAULT_HOST - port: int = DEFAULT_PORT - model: ModelFamily = ModelFamily.NVIDIA_GLINER - checkpoint: str | None = None - revision: str | None = None - - def __post_init__(self) -> None: - """Reject invalid transport values before model side effects.""" - if not 1 <= self.port <= 65535: - raise ValueError("port must be between 1 and 65535") - - @property - def resolved_checkpoint(self) -> str: - """Return the family default unless the user supplied an override.""" - if self.checkpoint: - return self.checkpoint - match self.model: - case ModelFamily.NVIDIA_GLINER: - return NVIDIA_GLINER_CHECKPOINT - case ModelFamily.GLINER2: - return GLINER2_CHECKPOINT - - -@dataclass(frozen=True, slots=True) -class Entity: - """One normalized entity in Anonymizer's detector response shape.""" - - text: str - label: str - start: int - end: int - score: float - - def as_dict(self) -> dict[str, str | int | float]: - """Serialize the entity in Anonymizer's required flat schema.""" - return {"text": self.text, "label": self.label, "start": self.start, "end": self.end, "score": self.score} - - -@dataclass(frozen=True, slots=True) -class DetectParams: - """Per-request inference settings that determine batching compatibility.""" - - labels: tuple[str, ...] - threshold: float - chunk_length: int - overlap: int - flat_ner: bool - inference_batch_size: int - - -class LocalRuntime(Protocol): - """Narrow common contract for local inference adapters.""" - - def infer(self, chunks: list[str], params: DetectParams) -> list[list[Entity]]: - """Detect normalized entities for each chunk.""" - - -if TYPE_CHECKING: - - class FastAPIRequest(Protocol): - """Request surface consumed by the OpenAI-compatible endpoint.""" - - async def json(self) -> object: - """Decode the request body.""" - -else: - FastAPIRequest = _fastapi.Request - - -class NvidiaModel(Protocol): - """Local API consumed from the optional original GLiNER package.""" - - def inference(self, **kwargs: object) -> object: - """Run one original GLiNER batch.""" - - -class Gliner2Model(Protocol): - """Local API consumed from the optional GLiNER2 package.""" - - def batch_extract_entities(self, chunks: list[str], labels: list[str], **kwargs: object) -> object: - """Run one GLiNER2 batch.""" - - -class NvidiaGlinerRuntime: - """Adapter for the original `gliner` local inference API.""" - - def __init__(self, model: NvidiaModel) -> None: - self._model = model - - def infer(self, chunks: list[str], params: DetectParams) -> list[list[Entity]]: - raw = self._model.inference( - texts=chunks, - labels=list(params.labels), - threshold=params.threshold, - flat_ner=params.flat_ner, - relations=[], - batch_size=params.inference_batch_size, - ) - return normalize_nvidia_output(raw) - - -class Gliner2Runtime: - """Adapter for GLiNER2's local batch extraction API.""" - - def __init__(self, model: Gliner2Model) -> None: - self._model = model - - def infer(self, chunks: list[str], params: DetectParams) -> list[list[Entity]]: - raw = self._model.batch_extract_entities( - chunks, - list(params.labels), - threshold=params.threshold, - include_confidence=True, - include_spans=True, - batch_size=params.inference_batch_size, - ) - return normalize_gliner2_output(raw) - - -def create_text_chunks(text: str, chunk_length: int, overlap: int) -> tuple[list[str], list[int]]: - """Split text into overlapping chunks while retaining their global offsets.""" - chunks: list[str] = [] - offsets: list[int] = [] - start = 0 - while start < len(text): - chunks.append(text[start : start + chunk_length]) - offsets.append(start) - if start + chunk_length >= len(text): - break - start += chunk_length - overlap - return chunks, offsets - - -def normalize_nvidia_output(raw: object) -> list[list[Entity]]: - """Convert original GLiNER dictionaries at the runtime boundary. - - Args: - raw: The `GLiNER.inference` batch result. - - Returns: - One normalized entity list per input chunk. - - Raises: - ValueError: If the runtime did not return GLiNER's documented batch shape. - """ - chunks = require_list(raw, "nvidia-gliner inference batch") - return [ - [normalize_nvidia_entity(entity) for entity in require_list(chunk, "nvidia-gliner chunk")] for chunk in chunks - ] - - -def normalize_gliner2_output(raw: object) -> list[list[Entity]]: - """Convert GLiNER2 entity records, including confidence and character spans. - - Args: - raw: The `GLiNER2.batch_extract_entities` result. - - Returns: - One normalized entity list per input chunk. - - Raises: - ValueError: If the runtime did not return a list of result mappings. - """ - return [ - normalize_gliner2_result(require_mapping(result, "gliner2 result")) - for result in require_list(raw, "gliner2 inference batch") - ] - - -def load_runtime(config: ServerConfig, device: str) -> LocalRuntime: - """Load the selected local runtime at the sole heavyweight side-effect boundary. - - Args: - config: Selected model family and checkpoint. - device: Local Torch device name. - - Returns: - A runtime adapter for inference. - """ - match config.model: - case ModelFamily.NVIDIA_GLINER: - gliner = importlib.import_module("gliner") - model = gliner.GLiNER.from_pretrained( - config.resolved_checkpoint, - map_location=device, - revision=config.revision, - ) - return NvidiaGlinerRuntime(model) - case ModelFamily.GLINER2: - gliner2 = importlib.import_module("gliner2") - checkpoint = config.resolved_checkpoint - if config.revision is not None: - hub = importlib.import_module("huggingface_hub") - checkpoint = hub.snapshot_download(repo_id=checkpoint, revision=config.revision) - model = gliner2.GLiNER2.from_pretrained(checkpoint, map_location=device) - return Gliner2Runtime(model) - - -def resolve_device() -> str: - """Choose an explicit DEVICE override or the best available local accelerator.""" - requested = os.getenv("DEVICE", "auto") - if requested != "auto": - return requested - torch = importlib.import_module("torch") - if torch.backends.mps.is_available(): - return "mps" - if torch.cuda.is_available(): - return "cuda" - return "cpu" - - -def finalize_entities(entities: list[Entity], *, flat_ner: bool) -> list[Entity]: - """Deduplicate overlap artifacts and optionally remove nested spans.""" - candidates = entities if flat_ner else remove_subset_entities(entities) - best: dict[tuple[str, str, int, int], Entity] = {} - for entity in candidates: - key = (entity.label, entity.text.strip().lower(), entity.start, entity.end) - if key not in best or entity.score > best[key].score: - best[key] = entity - return list(best.values()) - - -def remove_subset_entities(entities: list[Entity]) -> list[Entity]: - """Discard an entity wholly contained by a distinct larger entity.""" - return [ - entity - for entity in entities - if not any( - other != entity - and other.start <= entity.start - and other.end >= entity.end - and (other.start < entity.start or other.end > entity.end) - for other in entities - ) - ] - - -def detect_entities_for_texts(runtime: LocalRuntime, texts: list[str], params: DetectParams) -> list[list[Entity]]: - """Run all text chunks through one runtime batch and restore global offsets.""" - if not params.labels: - return [[] for _ in texts] - records = [ - (text_index, offset, chunk) - for text_index, text in enumerate(texts) - if text - for chunk, offset in zip(*create_text_chunks(text, params.chunk_length, params.overlap), strict=True) - ] - output = [[] for _ in texts] - if not records: - return output - inferred = runtime.infer([chunk for _, _, chunk in records], params) - for (text_index, offset, _), entities in zip(records, inferred, strict=True): - output[text_index].extend( - Entity(entity.text, entity.label, entity.start + offset, entity.end + offset, entity.score) - for entity in entities - ) - return [finalize_entities(entities, flat_ner=params.flat_ner) for entities in output] - - -@dataclass(slots=True) -class DetectJob: - """Queued request awaiting a shared inference call.""" - - text: str - params: DetectParams - future: asyncio.Future[list[Entity]] - - -class BatchDetector: - """Coalesce compatible requests while retaining one inference executor.""" - - def __init__(self, runtime: LocalRuntime) -> None: - self._runtime = runtime - self._queue: asyncio.Queue[DetectJob | None] = asyncio.Queue() - self._executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="gliner-infer") - self._worker_task: asyncio.Task[None] | None = None - - def start(self) -> None: - """Start the request-coalescing worker.""" - self._worker_task = asyncio.create_task(self._worker()) - - async def stop(self) -> None: - """Drain and stop the worker and its dedicated inference executor.""" - if self._worker_task is not None: - await self._queue.put(None) - await self._worker_task - self._executor.shutdown(wait=True) - - async def detect(self, text: str, params: DetectParams) -> list[Entity]: - """Queue one detection request or execute it serially when batching is off.""" - loop = asyncio.get_running_loop() - if not BATCH_MODE: - return ( - await loop.run_in_executor(self._executor, detect_entities_for_texts, self._runtime, [text], params) - )[0] - future: asyncio.Future[list[Entity]] = loop.create_future() - await self._queue.put(DetectJob(text, params, future)) - return await future - - async def _worker(self) -> None: - while first := await self._queue.get(): - jobs = [first] - deadline = asyncio.get_running_loop().time() + BATCH_WAIT_SECONDS - while len(jobs) < MAX_BATCH_REQUESTS: - try: - queued = await asyncio.wait_for( - self._queue.get(), max(0, deadline - asyncio.get_running_loop().time()) - ) - except TimeoutError: - break - if queued is None: - await self._queue.put(None) - break - jobs.append(queued) - await self._dispatch(jobs) - - async def _dispatch(self, jobs: list[DetectJob]) -> None: - groups: dict[DetectParams, list[DetectJob]] = {} - for job in jobs: - groups.setdefault(job.params, []).append(job) - loop = asyncio.get_running_loop() - for params, group in groups.items(): - try: - results = await loop.run_in_executor( - self._executor, detect_entities_for_texts, self._runtime, [job.text for job in group], params - ) - except Exception as exc: - for job in group: - if not job.future.done(): - job.future.set_exception(exc) - else: - for job, entities in zip(group, results, strict=True): - if not job.future.done(): - job.future.set_result(entities) - - -def normalize_nvidia_entity(raw: object) -> Entity: - """Normalize one original GLiNER entity dictionary.""" - mapping = require_mapping(raw, "nvidia-gliner entity") - return Entity( - str(mapping["text"]), - str(mapping["label"]), - coerce_int(mapping["start"], "nvidia-gliner start"), - coerce_int(mapping["end"], "nvidia-gliner end"), - coerce_float(mapping["score"], "nvidia-gliner score"), - ) - - -def normalize_gliner2_result(raw: Mapping[str, object]) -> list[Entity]: - """Normalize one GLiNER2 result mapping keyed by entity label.""" - entities = require_mapping(raw.get("entities"), "gliner2 entities") - return [ - normalize_gliner2_entity(label, item) - for label, values in entities.items() - for item in require_sequence(values, "gliner2 entity values") - ] - - -def normalize_gliner2_entity(label: object, raw: object) -> Entity: - """Normalize GLiNER2's span/confidence record for a named label.""" - entity = require_mapping(raw, "gliner2 entity") - span = entity.get("span") - if isinstance(span, Sequence) and not isinstance(span, str) and len(span) == 2: - start, end = span - else: - start, end = entity["start"], entity["end"] - confidence = entity.get("confidence", entity.get("score")) - if confidence is None: - raise ValueError("gliner2 entity is missing confidence") - return Entity( - str(entity["text"]), - str(label), - coerce_int(start, "gliner2 start"), - coerce_int(end, "gliner2 end"), - coerce_float(confidence, "gliner2 confidence"), - ) - - -def require_mapping(value: object, name: str) -> Mapping[str, object]: - """Validate an untyped runtime mapping at the adapter boundary.""" - if not isinstance(value, Mapping) or any(not isinstance(key, str) for key in value): - raise ValueError(f"unexpected {name} shape") - return cast(Mapping[str, object], value) - - -def require_sequence(value: object, name: str) -> Sequence[object]: - """Validate an untyped runtime sequence at the adapter boundary.""" - if not isinstance(value, Sequence) or isinstance(value, str): - raise ValueError(f"unexpected {name} shape") - return value - - -def require_list(value: object, name: str) -> list[object]: - """Validate a runtime list at the adapter boundary.""" - if not isinstance(value, list): - raise ValueError(f"unexpected {name} shape") - return cast(list[object], value) - - -def coerce_int(value: object, name: str) -> int: - """Convert a runtime numeric value or raise a boundary-specific error.""" - if isinstance(value, bool) or not isinstance(value, str | int | float): - raise ValueError(f"unexpected {name} value") - return int(value) - - -def coerce_float(value: object, name: str) -> float: - """Convert a runtime numeric value or raise a boundary-specific error.""" - if isinstance(value, bool) or not isinstance(value, str | int | float): - raise ValueError(f"unexpected {name} value") - return float(value) - - -def require_request_int(value: object, name: str) -> int: - """Accept only a JSON integer, excluding booleans and lossy coercions.""" - match value: - case bool(): - raise RequestValidationError(f"{name} must be an integer") - case int(): - return value - case _: - raise RequestValidationError(f"{name} must be an integer") - - -def require_request_float(value: object, name: str) -> float: - """Accept a finite JSON number without parsing strings or booleans.""" - match value: - case bool(): - raise RequestValidationError(f"{name} must be a number") - case int() | float(): - number = float(value) - case _: - raise RequestValidationError(f"{name} must be a number") - if not math.isfinite(number): - raise RequestValidationError(f"{name} must be finite") - return number - - -def require_request_bool(value: object, name: str) -> bool: - """Accept a JSON boolean without truthiness coercion.""" - match value: - case bool(): - return value - case _: - raise RequestValidationError(f"{name} must be a boolean") - - -def parse_detect_params(body: Mapping[str, object]) -> DetectParams: - """Parse and validate request options without performing I/O.""" - raw_labels = body.get("labels", []) - match raw_labels: - case list() if all(isinstance(label, str) for label in raw_labels): - labels = tuple(cast(str, label) for label in raw_labels) - case _: - raise RequestValidationError("labels must be a list of strings") - - threshold = require_request_float(body.get("threshold", 0.3), "threshold") - if not 0 <= threshold <= 1: - raise RequestValidationError("threshold must be between 0 and 1") - - chunk_length = require_request_int(body.get("chunk_length", DEFAULT_CHUNK_LENGTH), "chunk_length") - overlap = require_request_int(body.get("overlap", DEFAULT_OVERLAP), "overlap") - flat_ner = require_request_bool(body.get("flat_ner", DEFAULT_FLAT_NER), "flat_ner") - inference_batch_size = require_request_int(body.get("batch_size", DEFAULT_INFERENCE_BATCH_SIZE), "batch_size") - if inference_batch_size < 1: - raise RequestValidationError("batch_size must be >= 1") - validate_chunk_params(chunk_length, overlap) - return DetectParams(labels, threshold, chunk_length, overlap, flat_ner, inference_batch_size) - - -def extract_text(messages: object) -> str: - """Extract text from the final user message's string or multipart content.""" - if not isinstance(messages, list): - raise RequestValidationError("messages must be a list") - if not messages: - return "" - message = require_mapping(messages[-1], "message") - content = message.get("content", "") - if isinstance(content, str): - return content - if isinstance(content, list): - parts: list[str] = [] - for part in content: - text = require_mapping(part, "message part").get("text", "") - if not isinstance(text, str): - raise RequestValidationError("message part text must be a string") - parts.append(text) - return "".join(parts) - raise RequestValidationError("message content must be a string or list") - - -def validate_chunk_params(chunk_length: int, overlap: int) -> None: - """Reject invalid chunk settings before inference.""" - if chunk_length < 1: - raise RequestValidationError("chunk_length must be >= 1") - if overlap < 0 or overlap >= chunk_length: - raise RequestValidationError("overlap must be >= 0 and less than chunk_length") - - -state: ServerConfig | None = None -runtime: LocalRuntime | None = None -detector: BatchDetector | None = None -log = structlog.get_logger("gliner-server") - - -def configure_logging(log_format: LogFormat) -> None: - """Configure the selected human-readable or structured log renderer.""" - match log_format: - case LogFormat.PLAIN: - renderer = structlog.dev.ConsoleRenderer() - case LogFormat.JSON: - renderer = structlog.processors.JSONRenderer() - structlog.configure(processors=[renderer], wrapper_class=structlog.make_filtering_bound_logger(20)) - - -@asynccontextmanager -async def lifespan(_api: object) -> AsyncIterator[None]: - """Own the local runtime and its single inference worker for API lifetime.""" - global runtime, detector - if state is None: - raise RuntimeError("server configuration is not initialized") - device = resolve_device() - runtime = await asyncio.to_thread(load_runtime, state, device) - detector = BatchDetector(runtime) - detector.start() - log.info("server_ready", model=state.model, checkpoint=state.resolved_checkpoint, device=device) - try: - yield - finally: - if detector is not None: - await detector.stop() - - -api = _fastapi.FastAPI(lifespan=lifespan) -app = api - - -@api.get("/v1/models") -def list_models() -> dict[str, object]: - """Return the selected local checkpoint in OpenAI's model-list shape.""" - checkpoint = state.resolved_checkpoint if state else NVIDIA_GLINER_CHECKPOINT - return {"object": "list", "data": [{"id": checkpoint, "object": "model"}]} - - -@api.post("/v1/chat/completions") -async def chat_completions(request: FastAPIRequest) -> dict[str, object]: - """Detect requested entity labels and return Anonymizer's JSON-string content.""" - if detector is None: - raise _fastapi.HTTPException(status_code=503, detail="GLiNER model is not loaded") - try: - body = require_mapping(await request.json(), "request") - params = parse_detect_params(body) - text = extract_text(body.get("messages", [])) - except ValueError as exc: - raise _fastapi.HTTPException(status_code=422, detail=str(exc)) from exc - entities = await detector.detect(text, params) - content = json.dumps({"entities": [entity.as_dict() for entity in entities]}) - return { - "id": f"chatcmpl-{uuid.uuid4().hex[:12]}", - "object": "chat.completion", - "created": int(time.time()), - "model": str(body.get("model", state.resolved_checkpoint if state else NVIDIA_GLINER_CHECKPOINT)), - "choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, - } - - -cli = App(help="OpenAI-compatible local GLiNER server for Anonymizer.") - - -@cli.default -def main( - *, - host: str = DEFAULT_HOST, - port: int = DEFAULT_PORT, - model: ModelFamily = ModelFamily.NVIDIA_GLINER, - checkpoint: str | None = None, - revision: str | None = None, - log_format: LogFormat = LogFormat.PLAIN, -) -> None: - """Run the server without contacting any remote inference service. - - Args: - host: Bind address; use a private address unless protected by a proxy. - port: TCP listen port. - model: `nvidia-gliner` or `gliner2` local model family. - checkpoint: Optional Hugging Face checkpoint override for that family. - revision: Optional immutable Hugging Face model revision. - log_format: Human-readable `plain` logs or newline-delimited `json`. - """ - global state - try: - state = ServerConfig(host=host, port=port, model=model, checkpoint=checkpoint, revision=revision) - except ValueError as exc: - sys.stderr.write(f"error: {exc}\n") - raise SystemExit(125) from exc - configure_logging(log_format) - uvicorn.run(api, host=host, port=port) - - -if __name__ == "__main__": - cli() diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index 09b9d3b4..30d83d0e 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Explicit local-process and Docker effects for immutable run plans.""" +"""Explicit local-process effects for immutable run plans.""" from __future__ import annotations @@ -18,18 +18,13 @@ from inference_service_compiler.compiler import verify_plan from inference_service_compiler.models import ( - CachedModel, - CachedModels, CancellationReceipt, Capability, CapabilityProbeReceipt, - DockerHandle, - DockerRuntime, EntityDetection, Generation, LaunchReceipt, LocalProcessHandle, - LocalProcessRuntime, RunPlan, RuntimeDiagnostic, SecretEnvironmentVariable, @@ -45,33 +40,6 @@ def __init__(self, diagnostic: RuntimeDiagnostic) -> None: self.diagnostic = diagnostic -def discover_cached_models(cache_root: Path) -> CachedModels: - """List existing Hugging Face snapshots without creating or downloading files.""" - discovered: list[CachedModel] = [] - if cache_root.exists(): - for model_directory in sorted(cache_root.glob("models--*")): - repository = model_directory.name.removeprefix("models--").replace("--", "/") - for snapshot in sorted((model_directory / "snapshots").glob("*")): - if snapshot.is_dir(): - discovered.append( - CachedModel( - repository=repository, - revision=snapshot.name, - snapshot_path=str(snapshot), - ) - ) - return CachedModels(cache_root=str(cache_root), models=tuple(discovered)) - - -def default_cache_root() -> Path: - """Resolve the Hugging Face cache location without creating it.""" - if hub_cache := os.getenv("HF_HUB_CACHE"): - return Path(hub_cache) - if hf_home := os.getenv("HF_HOME"): - return Path(hf_home) / "hub" - return Path.home() / ".cache" / "huggingface" / "hub" - - def probe_endpoint( plan: RunPlan, *, @@ -127,7 +95,7 @@ def launch_plan( return LaunchReceipt( plan_digest=plan.plan_digest, launched_at=_now(), - shutdown_timeout_seconds=plan.intent.lifecycle.shutdown_timeout_seconds, + shutdown_timeout_seconds=plan.intent.local.shutdown_timeout_seconds, handle=handle, probe=probe, ) @@ -138,23 +106,19 @@ def _launch_handle( argv: tuple[str, ...], environment: dict[str, str], log_directory: Path, -) -> LocalProcessHandle | DockerHandle: - match plan.runtime: - case LocalProcessRuntime(): - return _launch_process(plan, argv, environment, log_directory) - case DockerRuntime(): - return _launch_docker(argv, environment) +) -> LocalProcessHandle: + return _launch_process(plan, argv, environment, log_directory) def _probe_or_cleanup( plan: RunPlan, - handle: LocalProcessHandle | DockerHandle, + handle: LocalProcessHandle, secret_values: Mapping[str, str], ) -> CapabilityProbeReceipt: try: probe = wait_for_readiness(plan, secret_values=secret_values, handle=handle) except RuntimeEffectError as exc: - cleanup_complete = _cleanup_handle(handle, plan.intent.lifecycle.shutdown_timeout_seconds) + cleanup_complete = _cleanup_handle(handle, plan.intent.local.shutdown_timeout_seconds) raise RuntimeEffectError( exc.diagnostic.model_copy( update={ @@ -164,7 +128,7 @@ def _probe_or_cleanup( ) ) from exc if not probe.passed: - cleanup_complete = _cleanup_handle(handle, plan.intent.lifecycle.shutdown_timeout_seconds) + cleanup_complete = _cleanup_handle(handle, plan.intent.local.shutdown_timeout_seconds) raise RuntimeEffectError( RuntimeDiagnostic( code="capability-mismatch", @@ -188,7 +152,7 @@ def inspect_run(launch: LaunchReceipt) -> StatusReceipt: def cancel_run(launch: LaunchReceipt) -> CancellationReceipt: - """Stop the exact process group or container recorded by a launch receipt.""" + """Stop the exact process group recorded by a launch receipt.""" handle = launch.handle if not is_handle_running(handle): return CancellationReceipt( @@ -208,33 +172,23 @@ def cancel_run(launch: LaunchReceipt) -> CancellationReceipt: ) -def is_handle_running(handle: LocalProcessHandle | DockerHandle) -> bool: +def is_handle_running(handle: LocalProcessHandle) -> bool: """Check the external identity while guarding against Linux PID reuse.""" - match handle: - case LocalProcessHandle(): - current_marker = read_process_start_marker(handle.pid) - if handle.start_marker is not None and current_marker != handle.start_marker: - return False - if read_process_state(handle.pid) == "Z": - return False - try: - os.kill(handle.pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True - return True - case DockerHandle(): - completed = subprocess.run( - ["docker", "inspect", "--format", "{{.State.Running}}", handle.container_id], - capture_output=True, - text=True, - check=False, - ) - return completed.returncode == 0 and completed.stdout.strip().lower() == "true" + current_marker = read_process_start_marker(handle.pid) + if handle.start_marker is not None and current_marker != handle.start_marker: + return False + if read_process_state(handle.pid) == "Z": + return False + try: + os.kill(handle.pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True -def _cleanup_handle(handle: LocalProcessHandle | DockerHandle, timeout_seconds: float) -> bool: +def _cleanup_handle(handle: LocalProcessHandle, timeout_seconds: float) -> bool: if not is_handle_running(handle): return True _outcome, cleanup_complete = _stop_running_handle(handle, timeout_seconds) @@ -242,62 +196,43 @@ def _cleanup_handle(handle: LocalProcessHandle | DockerHandle, timeout_seconds: def _stop_running_handle( - handle: LocalProcessHandle | DockerHandle, + handle: LocalProcessHandle, timeout_seconds: float, ) -> tuple[Literal["terminated", "forced"], bool]: - match handle: - case LocalProcessHandle(): - try: - os.killpg(handle.process_group_id, signal.SIGTERM) - except ProcessLookupError: - return "terminated", True - deadline = time.monotonic() + timeout_seconds - running = True - while time.monotonic() < deadline: - running = is_handle_running(handle) - if not running: - break - time.sleep(0.1) - outcome: Literal["terminated", "forced"] = "terminated" - if running: - try: - os.killpg(handle.process_group_id, signal.SIGKILL) - except ProcessLookupError: - return "forced", True - running = is_handle_running(handle) - outcome = "forced" - return outcome, not running - case DockerHandle(): - completed = subprocess.run( - ["docker", "stop", "--time", str(int(timeout_seconds)), handle.container_id], - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - raise RuntimeEffectError( - RuntimeDiagnostic( - code="docker-cancel-failed", - message=completed.stderr.strip() or "docker stop failed", - known_effects=(handle.external_id,), - cleanup_complete=False, - ) - ) - return "terminated", True + try: + os.killpg(handle.process_group_id, signal.SIGTERM) + except ProcessLookupError: + return "terminated", True + deadline = time.monotonic() + timeout_seconds + running = True + while time.monotonic() < deadline: + running = is_handle_running(handle) + if not running: + break + time.sleep(0.1) + outcome: Literal["terminated", "forced"] = "terminated" + if running: + try: + os.killpg(handle.process_group_id, signal.SIGKILL) + except ProcessLookupError: + return "forced", True + running = is_handle_running(handle) + outcome = "forced" + return outcome, not running def wait_for_readiness( plan: RunPlan, *, secret_values: Mapping[str, str] | None = None, - handle: LocalProcessHandle | DockerHandle | None = None, + handle: LocalProcessHandle | None = None, ) -> CapabilityProbeReceipt: """Poll the declared readiness contract until it passes or times out.""" deadline = time.monotonic() + plan.readiness.timeout_seconds last_error: RuntimeEffectError | None = None while time.monotonic() < deadline: if handle is not None and not is_handle_running(handle): - log_hint = f"; inspect {handle.stderr_path}" if isinstance(handle, LocalProcessHandle) else "" + log_hint = f"; inspect {handle.stderr_path}" raise RuntimeEffectError( RuntimeDiagnostic( code="launch-exited", @@ -467,19 +402,5 @@ def _launch_process( ) -def _launch_docker(argv: tuple[str, ...], environment: dict[str, str]) -> DockerHandle: - completed = subprocess.run(argv, env=environment, capture_output=True, text=True, check=False) - if completed.returncode != 0: - raise RuntimeEffectError( - RuntimeDiagnostic(code="docker-launch-failed", message=completed.stderr.strip() or "docker run failed") - ) - container_id = completed.stdout.strip() - if not container_id: - raise RuntimeEffectError( - RuntimeDiagnostic(code="docker-launch-failed", message="docker run returned no container identity") - ) - return DockerHandle(external_id=container_id, container_id=container_id) - - def _now() -> str: return datetime.now(UTC).isoformat() diff --git a/tools/inference_service_profiles/gliner2.toml b/tools/inference_service_profiles/gliner2.toml index 6376e6d9..391db093 100644 --- a/tools/inference_service_profiles/gliner2.toml +++ b/tools/inference_service_profiles/gliner2.toml @@ -1,4 +1,4 @@ -schema_version = "inference-service.intent/v1" +schema_version = "inference-service.intent/v2" [task] kind = "entity-detection" @@ -7,29 +7,21 @@ offsets = true scores = true [model] -kind = "hugging-face" model_id = "fastino/gliner2-privacy-filter-PII-multi" revision = "59894c087cb2923b01f337d4ee72f6ff84d5bdd6" -[engine] -kind = "vllm" +[vllm] python_executable = ".venv/bin/python" gpu_memory_utilization = 0.85 max_model_len = 512 -[engine.factory] +[vllm.factory] plugin = "deberta_gliner2" prepared_model_root = "/tmp/anonymizer-vllm-factory" -[placement] -kind = "local-process" +[local] host = "127.0.0.1" port = 8002 -[access] -kind = "direct" - -[lifecycle] -kind = "managed" startup_timeout_seconds = 300 shutdown_timeout_seconds = 30 diff --git a/tools/inference_service_profiles/gpt-oss-120b.toml b/tools/inference_service_profiles/gpt-oss-120b.toml index b6ae18ef..7c4af546 100644 --- a/tools/inference_service_profiles/gpt-oss-120b.toml +++ b/tools/inference_service_profiles/gpt-oss-120b.toml @@ -1,16 +1,14 @@ -schema_version = "inference-service.intent/v1" +schema_version = "inference-service.intent/v2" [task] kind = "generation" chat = true [model] -kind = "hugging-face" model_id = "openai/gpt-oss-120b" revision = "b5c939de8f754692c1647ca79fbf85e8c1e70f8a" -[engine] -kind = "vllm" +[vllm] python_executable = ".venv/bin/python" served_model_name = "gpt-oss-120b-local" gpu_memory_utilization = 0.95 @@ -19,15 +17,9 @@ max_num_seqs = 8 enable_prefix_caching = true async_scheduling = true -[placement] -kind = "local-process" +[local] host = "127.0.0.1" port = 8000 -[access] -kind = "direct" - -[lifecycle] -kind = "managed" startup_timeout_seconds = 1800 shutdown_timeout_seconds = 60 diff --git a/tools/inference_service_profiles/gpt-oss-20b.toml b/tools/inference_service_profiles/gpt-oss-20b.toml index e3153eb6..d9d65d92 100644 --- a/tools/inference_service_profiles/gpt-oss-20b.toml +++ b/tools/inference_service_profiles/gpt-oss-20b.toml @@ -1,16 +1,14 @@ -schema_version = "inference-service.intent/v1" +schema_version = "inference-service.intent/v2" [task] kind = "generation" chat = true [model] -kind = "hugging-face" model_id = "openai/gpt-oss-20b" revision = "6cee5e81ee83917806bbde320786a8fb61efebee" -[engine] -kind = "vllm" +[vllm] python_executable = ".venv/bin/python" served_model_name = "gpt-oss-20b-local" gpu_memory_utilization = 0.85 @@ -19,15 +17,9 @@ max_num_seqs = 16 enable_prefix_caching = true async_scheduling = true -[placement] -kind = "local-process" +[local] host = "127.0.0.1" port = 8000 -[access] -kind = "direct" - -[lifecycle] -kind = "managed" startup_timeout_seconds = 1200 shutdown_timeout_seconds = 60 diff --git a/tools/inference_service_profiles/nemotron-3.5-lightning.toml b/tools/inference_service_profiles/nemotron-3.5-lightning.toml index e16423b6..cf924252 100644 --- a/tools/inference_service_profiles/nemotron-3.5-lightning.toml +++ b/tools/inference_service_profiles/nemotron-3.5-lightning.toml @@ -1,16 +1,14 @@ -schema_version = "inference-service.intent/v1" +schema_version = "inference-service.intent/v2" [task] kind = "generation" chat = true [model] -kind = "hugging-face" model_id = "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16" revision = "33268dc8a6da85a56be2b12241453e4e1237bbe1" -[engine] -kind = "vllm" +[vllm] python_executable = ".venv/bin/python" served_model_name = "nemotron-3.5-lightning-local" gpu_memory_utilization = 0.88 @@ -23,15 +21,9 @@ mamba_ssm_cache_dtype = "float16" enable_mamba_cache_stochastic_rounding = true mamba_cache_philox_rounds = 5 -[placement] -kind = "local-process" +[local] host = "127.0.0.1" port = 8000 -[access] -kind = "direct" - -[lifecycle] -kind = "managed" startup_timeout_seconds = 1800 shutdown_timeout_seconds = 60 diff --git a/tools/inference_service_profiles/nvidia-gliner.toml b/tools/inference_service_profiles/nvidia-gliner.toml index 2a0de58c..f62357e3 100644 --- a/tools/inference_service_profiles/nvidia-gliner.toml +++ b/tools/inference_service_profiles/nvidia-gliner.toml @@ -1,4 +1,4 @@ -schema_version = "inference-service.intent/v1" +schema_version = "inference-service.intent/v2" [task] kind = "entity-detection" @@ -7,29 +7,21 @@ offsets = true scores = true [model] -kind = "hugging-face" model_id = "nvidia/gliner-pii" revision = "bd23e8ef4425fd04e34c5204ab49ffaa706eae79" -[engine] -kind = "vllm" +[vllm] python_executable = ".venv/bin/python" gpu_memory_utilization = 0.85 max_model_len = 512 -[engine.factory] +[vllm.factory] plugin = "deberta_gliner" prepared_model_root = "/tmp/anonymizer-vllm-factory" -[placement] -kind = "local-process" +[local] host = "127.0.0.1" port = 8001 -[access] -kind = "direct" - -[lifecycle] -kind = "managed" startup_timeout_seconds = 300 shutdown_timeout_seconds = 30 diff --git a/tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml b/tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml index d89dfe43..a1b50f91 100644 --- a/tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml +++ b/tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml @@ -1,16 +1,14 @@ -schema_version = "inference-service.intent/v1" +schema_version = "inference-service.intent/v2" [task] kind = "generation" chat = true [model] -kind = "hugging-face" model_id = "Qwen/Qwen3-30B-A3B-Instruct-2507" revision = "0d7cf23991f47feeb3a57ecb4c9cee8ea4a17bfe" -[engine] -kind = "vllm" +[vllm] python_executable = ".venv/bin/python" served_model_name = "qwen3-30b-a3b-instruct-local" gpu_memory_utilization = 0.90 @@ -19,15 +17,9 @@ max_num_seqs = 16 enable_prefix_caching = true async_scheduling = true -[placement] -kind = "local-process" +[local] host = "127.0.0.1" port = 8000 -[access] -kind = "direct" - -[lifecycle] -kind = "managed" startup_timeout_seconds = 1800 shutdown_timeout_seconds = 60 diff --git a/tools/inference_service_profiles/vllm-local.toml b/tools/inference_service_profiles/vllm-local.toml index 06d5399f..3f9aec39 100644 --- a/tools/inference_service_profiles/vllm-local.toml +++ b/tools/inference_service_profiles/vllm-local.toml @@ -1,30 +1,22 @@ -schema_version = "inference-service.intent/v1" +schema_version = "inference-service.intent/v2" [task] kind = "generation" chat = true [model] -kind = "hugging-face" model_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" revision = "fe8a4ea1ffedaf415f4da2f062534de366a451e6" -[engine] -kind = "vllm" +[vllm] python_executable = ".venv/bin/python" served_model_name = "anonymizer-local" gpu_memory_utilization = 0.85 max_model_len = 2048 -[placement] -kind = "local-process" +[local] host = "127.0.0.1" port = 8000 -[access] -kind = "direct" - -[lifecycle] -kind = "managed" startup_timeout_seconds = 600 shutdown_timeout_seconds = 30 From f62ee55b66fc4371b06f9b312c3c248139a59ae7 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 13 Aug 2026 04:58:09 +0000 Subject: [PATCH 10/28] docs: add local model container deployment Signed-off-by: Aaron Gonzales --- .dockerignore | 14 + README.md | 10 +- docs/concepts/inference-services.md | 291 ++++++++++++++---- docs/concepts/models.md | 8 +- docs/concepts/self-hosting-gliner.md | 17 +- .../posts/self-hosted-anonymizer-b300.md | 38 +-- mkdocs.yml | 2 +- skills/anonymizer/SKILL.md | 4 +- tools/inference_service.Dockerfile | 26 ++ 9 files changed, 313 insertions(+), 97 deletions(-) create mode 100644 .dockerignore create mode 100644 tools/inference_service.Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..339c1356 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +.git +.venv +.cache +.pytest_cache +.ruff_cache +.ty_cache +.anonymizer-artifacts +.inference-service-runs +site +**/__pycache__ +**/*.pyc diff --git a/README.md b/README.md index 1a631271..bd76d6ab 100644 --- a/README.md +++ b/README.md @@ -156,9 +156,9 @@ make install-pre-commit # Install pre-commit hooks ### Local inference services -Use the source-tree inference service compiler to create immutable plans and +Use the source-tree local-model deployment tool to create immutable plans and managed launch receipts for GLiNER or GLiNER2 through the pinned external vLLM -Factory project, the native detector fallback, and vLLM generation: +Factory project, or for vLLM generation: ```bash uv run tools/inference_service.py compile --profile tools/inference_service_profiles/nvidia-gliner.toml --source-revision 3f68c145 --output plan.json @@ -168,9 +168,9 @@ uv run tools/inference_service.py cancel --receipt launch.json ``` The tool is not part of the wheel and does not attach to externally owned -endpoints. The [local inference service guide](docs/concepts/inference-services.md) -covers typed TOML profiles, GPU-host setup, and capability -probes, and Anonymizer provider configuration. +endpoints. The [local model deployment guide](docs/concepts/inference-services.md) +covers typed TOML profiles, direct GPU-host setup, a container-based deployment, +capability probes, cleanup, and Anonymizer provider configuration. --- diff --git a/docs/concepts/inference-services.md b/docs/concepts/inference-services.md index 13ca7801..6ac4966e 100644 --- a/docs/concepts/inference-services.md +++ b/docs/concepts/inference-services.md @@ -1,81 +1,266 @@ -# Run Local Inference Services +# Deploy Local Models -The source tree provides `tools/inference_service.py` for compiling and managing -one domain: a local-process vLLM server. Compilation is pure and produces a -versioned, digest-protected plan; launch, probe, inspect, and cancel consume that -plan rather than re-reading a profile. +NeMo Anonymizer can run its detector and generation roles against models on +your own NVIDIA GPU. The repository includes pinned profiles and a lifecycle +tool for this purpose. The tool compiles a profile into an immutable plan, +starts the service, proves its API contract, and records enough process identity +to inspect or stop it later. -Install the optional local-model dependencies on a Linux GPU host: +This path suits development workstations, on-premises servers, and isolated +environments where text must stay on local infrastructure. It manages one +deployment shape: a vLLM process on the machine where the tool runs. You can +run that process directly on the host or inside the supplied GPU container. + +The tool lives under `tools/` and is not installed with the +`nemo-anonymizer` wheel. Start from a source checkout. + +## Choose models + +Seven profiles ship in `tools/inference_service_profiles/`: + +| Profile | Endpoint model name | Role and hardware guidance | +| --- | --- | --- | +| `nvidia-gliner.toml` | `nvidia/gliner-pii` | Default PII detector; can share an 80 GB GPU with a 20B or 30B generator | +| `gliner2.toml` | `fastino/gliner2-privacy-filter-PII-multi` | Alternative multilingual PII detector | +| `vllm-local.toml` | `anonymizer-local` | TinyLlama lifecycle smoke test | +| `gpt-oss-20b.toml` | `gpt-oss-20b-local` | Compact GPT-OSS generation | +| `gpt-oss-120b.toml` | `gpt-oss-120b-local` | GPT-OSS generation on a dedicated 80 GB GPU | +| `qwen3-30b-a3b-instruct.toml` | `qwen3-30b-a3b-instruct-local` | Multilingual generation | +| `nemotron-3.5-lightning.toml` | `nemotron-3.5-lightning-local` | High-throughput generation on an 80 GB GPU | + +Each profile pins its Hugging Face revision. Generation profiles use stock +vLLM. The two detector profiles use the pinned external +[vLLM Factory](https://github.com/latenceainew/vllm-factory) integration and +Anonymizer's OpenAI-compatible detector adapter. + +GPU memory also depends on the architecture, context length, concurrency, and +other processes. Lower `max_model_len`, `max_num_seqs`, or +`gpu_memory_utilization` when vLLM cannot reserve its KV cache. Do not co-host +the GPT-OSS 120B profile with another model on an 80 GB GPU. + +## Understand the lifecycle + +The workflow has two durable files: + +```text +profile.toml -> compile -> plan.json -> launch -> launch.json + | | + +-> probe +-> inspect + +-> cancel +``` + +`compile` has no runtime effects. The plan contains the full command, endpoint, +dependencies, compatibility evidence, and a SHA-256 digest. Runtime commands +verify that digest before acting. + +`launch` waits for `/v1/models` and a task-specific request. A generation +service must return a chat completion. A detector must demonstrate dynamic +labels, offsets, and scores. The launch receipt records the process group and +Linux start marker so later commands do not signal a reused PID. + +The v2 profile schema has four sections: + +- `[task]` selects generation or entity detection and its required capabilities. +- `[model]` pins the Hugging Face model and optional LoRA adapter. +- `[vllm]` sets the served name, memory limits, parallelism, caching, Mamba controls, authentication source, and optional Factory plugin. +- `[local]` sets the bind address, port, startup timeout, and shutdown timeout. + +## Deploy on the GPU host + +### Install + +Install [uv](https://docs.astral.sh/uv/) and sync the local-model dependency +group with Python 3.12: ```bash uv sync --python 3.12 --group dev --group local-models -python -m vllm_factory.compat.doctor +uv run --python 3.12 python -m vllm_factory.compat.doctor nvidia-smi ``` -The group pins vLLM 0.27.1, the external vLLM Factory revision, and the CUDA -compiler wheels needed by the Nemotron profile. +The lockfile pins vLLM 0.27.1, vLLM Factory, and the CUDA compiler wheels used +by the Nemotron FlashInfer profile. -## Profiles +### Compile and launch -Seven pinned profiles ship in `tools/inference_service_profiles/`. Generation -profiles use Hugging Face vLLM. `nvidia-gliner.toml` and `gliner2.toml` use the -pinned NVIDIA vLLM Factory integration for entity detection. Detection requires -the Factory section; stock vLLM generation must not include it. +Run all commands from the repository root. This example starts NVIDIA GLiNER +on `127.0.0.1:8001`: -| Profile | Served model | Use | -| --- | --- | --- | -| `vllm-local.toml` | `anonymizer-local` | Small lifecycle smoke tests | -| `gpt-oss-20b.toml` | `gpt-oss-20b-local` | Compact GPT-OSS development | -| `gpt-oss-120b.toml` | `gpt-oss-120b-local` | Dedicated 80 GB GPU | -| `qwen3-30b-a3b-instruct.toml` | `qwen3-30b-a3b-instruct-local` | Multilingual generation | -| `nemotron-3.5-lightning.toml` | `nemotron-3.5-lightning-local` | High-throughput generation | -| `nvidia-gliner.toml` | model ID | NVIDIA GLiNER detection | -| `gliner2.toml` | model ID | GLiNER2 detection | +```bash +uv run --python 3.12 python tools/inference_service.py compile \ + --profile tools/inference_service_profiles/nvidia-gliner.toml \ + --source-revision "$(git rev-parse HEAD)" \ + --output gliner-plan.json + +uv run --python 3.12 python tools/inference_service.py launch \ + --plan gliner-plan.json \ + --output gliner-launch.json \ + --log-directory .inference-service-runs +``` + +The command returns after both probes pass. Keep `gliner-launch.json`; it owns +the process identity used for inspection and cleanup. + +### Operate the service + +```bash +uv run --python 3.12 python tools/inference_service.py inspect \ + --receipt gliner-launch.json + +uv run --python 3.12 python tools/inference_service.py probe \ + --plan gliner-plan.json + +curl -sf http://127.0.0.1:8001/v1/models | python -m json.tool + +uv run --python 3.12 python tools/inference_service.py cancel \ + --receipt gliner-launch.json +``` + +`cancel` sends `SIGTERM` to the receipt-owned process group, waits for the +profile's shutdown timeout, and uses `SIGKILL` if the group remains alive. + +## Deploy in a GPU container -Memory requirements also depend on the GPU, driver, context length, and -concurrency. Reduce `max_model_len` or `max_num_seqs` if vLLM cannot reserve its -KV cache. Do not co-host the 120B GPT-OSS profile on the GPU used for detection. +The supplied container image installs the same locked local-model environment. +The container stays alive as the deployment boundary. The lifecycle tool still +manages a local process inside that boundary, so plans and receipts keep the +same schema as a host deployment. -## Lifecycle +### Prerequisites + +Install Docker, the NVIDIA driver, and +[NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). +Confirm that Docker can expose the GPU: + +```bash +docker run --rm --gpus all \ + nvidia/cuda:13.0.2-base-ubuntu24.04 nvidia-smi +``` + +### Build the image + +From the repository root: + +```bash +docker build \ + --file tools/inference_service.Dockerfile \ + --tag nemo-anonymizer-local-models:dev \ + . +``` + +The build uses the repository lockfile. It downloads several large GPU wheels, +so allow enough disk space for the image and model cache. + +### Start the deployment container + +Create host directories for receipts, logs, prepared Factory models, and the +Hugging Face cache: + +```bash +mkdir -p .local-model-deployment/state +mkdir -p .local-model-deployment/factory +mkdir -p "$HOME/.cache/huggingface" + +docker run --detach \ + --name anonymizer-local-models \ + --gpus all \ + --ipc host \ + --network host \ + --volume "$PWD/.local-model-deployment/state:/state" \ + --volume "$PWD/.local-model-deployment/factory:/tmp/anonymizer-vllm-factory" \ + --volume "$HOME/.cache/huggingface:/models/huggingface" \ + nemo-anonymizer-local-models:dev +``` + +Host networking is required because the pinned profiles bind to `127.0.0.1`. +It is supported by Docker Engine on Linux. The profile port must be free on the +host. The default profiles do not authenticate requests, so keep the bind local +or place the service behind authenticated TLS before exposing it. + +### Compile and launch inside the container ```bash -uv run tools/inference_service.py compile \ +SOURCE_REVISION="$(git rev-parse HEAD)" +docker exec anonymizer-local-models \ + python tools/inference_service.py compile \ --profile tools/inference_service_profiles/nvidia-gliner.toml \ - --source-revision "$(git rev-parse HEAD)" --output plan.json -uv run tools/inference_service.py launch --plan plan.json --output launch.json -uv run tools/inference_service.py inspect --receipt launch.json -uv run tools/inference_service.py probe --plan plan.json -uv run tools/inference_service.py cancel --receipt launch.json + --source-revision "$SOURCE_REVISION" \ + --output /state/gliner-plan.json + +docker exec anonymizer-local-models \ + python tools/inference_service.py launch \ + --plan /state/gliner-plan.json \ + --output /state/gliner-launch.json \ + --log-directory /state/logs ``` -The `[local]` section holds host, port, and bounded startup/shutdown timeouts. -The `[vllm]` section holds tensor parallelism, memory and model limits, API-key -environment source, LoRA, eager/prefix/async controls, and Mamba controls. -Secrets remain symbolic in plans and are read from their named environment -variable only at launch. Probes use `/v1/models` plus a task-aware chat payload; -the generation probe accepts reasoning-aware GPT-OSS responses. +The mounted `/state` directory makes the plan, launch receipt, and logs visible +on the host. Operate the service through `docker exec`: + +```bash +docker exec anonymizer-local-models \ + python tools/inference_service.py inspect \ + --receipt /state/gliner-launch.json -## GLiNER through vLLM Factory +docker exec anonymizer-local-models \ + python tools/inference_service.py probe \ + --plan /state/gliner-plan.json + +curl -sf http://127.0.0.1:8001/v1/models | python -m json.tool +``` -Factory-backed detection keeps vLLM Factory's pooling endpoint and adds the -OpenAI-compatible chat contract used by Anonymizer. The adapter preserves -dynamic labels, offsets, scores, and overlapping character chunks. Model -preparation uses the profile's pinned Hugging Face revision. +### Stop cleanly + +Cancel each managed service before removing the container: + +```bash +docker exec anonymizer-local-models \ + python tools/inference_service.py cancel \ + --receipt /state/gliner-launch.json + +docker stop anonymizer-local-models +docker rm anonymizer-local-models +``` + +Stopping the container first removes the runtime boundary before the tool can +write a cancellation receipt. Use `cancel` first when lifecycle evidence or +graceful model shutdown matters. ## Connect Anonymizer -Use the plan endpoint and served model name in a custom DataDesigner provider -and model configuration. Custom model configuration replaces Anonymizer's -bundled model pool, so retain every alias required by the roles you use. See -[Custom models](models.md#custom-models) for the role map. +Add the local endpoint to a provider file: + +```yaml title="providers.yaml" +providers: + - name: local-gliner + endpoint: http://127.0.0.1:8001/v1 + provider_type: openai + api_key: EMPTY +``` + +Then route the detector alias to that provider in your model configuration: + +```yaml title="models.yaml" +model_configs: + - alias: gliner-pii-detector + model: nvidia/gliner-pii + provider: local-gliner + inference_parameters: + max_parallel_requests: 8 + timeout: 120 +``` + +Custom `model_configs` replaces the full bundled model pool. Copy the bundled +configuration and change only the roles you are deploying locally, or define +every alias needed by the selected pipeline. See +[Custom models](models.md#custom-models) for role mapping and complete examples. -Compilation proves static compatibility. The launch probe proves the observed -endpoint contract. Run Anonymizer preview and evaluation before trusting a new -model for privacy or utility. +For the detector request and response contract, chunking behavior, and a live +PII request, see [Self-hosting GLiNER](self-hosting-gliner.md). -Docker placement, native Transformers GLiNER serving, and cache discovery are -not supported by this tool. +Compilation proves static compatibility. The launch probe proves the endpoint +shape. Run Anonymizer preview and evaluation before accepting a model for a +privacy-sensitive workload. diff --git a/docs/concepts/models.md b/docs/concepts/models.md index 42873b9c..40b7dbf2 100644 --- a/docs/concepts/models.md +++ b/docs/concepts/models.md @@ -36,10 +36,10 @@ Each pipeline stage has a **role** mapped to one of these aliases. See the full Pass `model_providers` when you need a non-default endpoint — for example OpenAI, OpenRouter, a local GLiNER server, or an internal inference deployment. Plain `Anonymizer()` already uses bundled [build.nvidia.com](https://build.nvidia.com) settings; override only when your models point at a different provider name or URL. -For managed GLiNER through vLLM Factory, the native detector fallback, and -vLLM generation endpoints, see [Run local inference services](inference-services.md). -That guide covers immutable plans, local processes, and capability -receipts, and provider configuration. +For managed GLiNER through vLLM Factory and vLLM generation endpoints, see +[Deploy local models](inference-services.md). That guide covers host and GPU +container deployment, immutable plans, capability receipts, cleanup, and +provider configuration. Set your API keys first: diff --git a/docs/concepts/self-hosting-gliner.md b/docs/concepts/self-hosting-gliner.md index 01363c93..f780c75e 100644 --- a/docs/concepts/self-hosting-gliner.md +++ b/docs/concepts/self-hosting-gliner.md @@ -8,11 +8,10 @@ By default, Anonymizer's entity detection stage calls the hosted `nvidia/gliner- The default NVIDIA GLiNER model is small enough to share a GPU with a local LLM. The optional GLiNER2 PII model is also fully local. The pinned reference profiles serve both models through vLLM 0.27.1 and the external -[vLLM Factory](https://github.com/latenceainew/vllm-factory) project. A native -CPU, MPS, or GPU fallback remains available for custom profiles. +[vLLM Factory](https://github.com/latenceainew/vllm-factory) project. The characterized services live inside the source-tree -[inference service compiler](inference-services.md). They are **not** installed +[local-model deployment tool](inference-services.md). They are **not** installed with `pip install nemo-anonymizer`; compile and launch them from a source checkout. @@ -91,7 +90,7 @@ Native Transformers GLiNER fallback serving is intentionally not included. --- -## Running it +## Deploying it !!! note "Source checkout only" @@ -121,10 +120,8 @@ the network; inference stays local after setup. ### Start the server Compile the pinned vLLM Factory GLiNER TOML profile and launch the resulting -plan as shown in -[Run local inference services](inference-services.md#gliner-through-vllm-factory). -The profile keeps the model checkpoint, engine, factory plugin, placement, -access, and managed lifecycle separate. NVIDIA GLiNER uses +plan as shown in [Deploy local models](inference-services.md). That guide has +complete host and GPU container workflows. NVIDIA GLiNER uses `deberta_gliner`; GLiNER2 uses `deberta_gliner2`. Launch writes a versioned receipt only after the model-list and detection @@ -135,8 +132,8 @@ The model families do not use identical label vocabularies. The request example The reference profiles have **no authentication**. The default bind address is `127.0.0.1` so detection traffic stays on localhost. Set `api_key_env` in the -engine or place the endpoint behind authenticated TLS before exposing it to -another host. +`[vllm]` section or place the endpoint behind authenticated TLS before exposing +it to another host. Verify the server is reachable: diff --git a/docs/devnotes/posts/self-hosted-anonymizer-b300.md b/docs/devnotes/posts/self-hosted-anonymizer-b300.md index ac7ee1f8..c3d5de3c 100644 --- a/docs/devnotes/posts/self-hosted-anonymizer-b300.md +++ b/docs/devnotes/posts/self-hosted-anonymizer-b300.md @@ -133,13 +133,13 @@ The `CUDA_ROOT` path above is specific to the Brev B300 SXM6 environment used fo `--gpu-memory-utilization 0.45` was a conservative co-location setting, not a compute throttle. In vLLM it controls the GPU memory budget for model weights and KV cache. Qwen could use more memory if the run needed a larger KV cache, but this setting left headroom for the GLiNER server on the same GPU and still completed the measured batches with zero failures. -GLiNER ran on the same machine. Current reruns use the source-tree inference -service compiler described in [Self-hosting GLiNER](../../concepts/self-hosting-gliner.md), -which records the exact model, engine, placement, batch environment, endpoint, -and process identity in versioned plans and receipts. +GLiNER ran on the same machine. Current reruns use the source-tree local-model +deployment tool described in [Self-hosting GLiNER](../../concepts/self-hosting-gliner.md). +It records the exact model, vLLM settings, endpoint, and process identity in +versioned plans and receipts. The current detector path uses vLLM Factory. ```toml title="gliner-b300.toml" -schema_version = "inference-service.intent/v1" +schema_version = "inference-service.intent/v2" [task] kind = "entity-detection" @@ -148,37 +148,31 @@ offsets = true scores = true [model] -kind = "hugging-face" model_id = "nvidia/gliner-pii" revision = "bd23e8ef4425fd04e34c5204ab49ffaa706eae79" -[engine] -kind = "native-gliner" -family = "nvidia-gliner" -device = "cuda" -max_batch_requests = 64 -batch_wait_ms = 10 +[vllm] +python_executable = ".venv/bin/python" +gpu_memory_utilization = 0.45 +max_model_len = 512 -[placement] -kind = "local-process" +[vllm.factory] +plugin = "deberta_gliner" +prepared_model_root = "/tmp/anonymizer-vllm-factory" + +[local] host = "127.0.0.1" port = 9000 - -[access] -kind = "direct" - -[lifecycle] -kind = "managed" startup_timeout_seconds = 300 shutdown_timeout_seconds = 30 ``` ```bash -uv run tools/inference_service.py compile \ +uv run --python 3.12 python tools/inference_service.py compile \ --profile gliner-b300.toml \ --source-revision 3f68c145 \ --output gliner-b300-plan.json -uv run tools/inference_service.py launch \ +uv run --python 3.12 python tools/inference_service.py launch \ --plan gliner-b300-plan.json \ --output gliner-b300-launch.json \ --log-directory logs diff --git a/mkdocs.yml b/mkdocs.yml index e2e95235..f4ee3b08 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -161,7 +161,7 @@ nav: - Choosing a Strategy: concepts/choosing-a-strategy.md - Evaluation: concepts/evaluation.md - Self-hosting GLiNER: concepts/self-hosting-gliner.md - - Run Local Inference Services: concepts/inference-services.md + - Deploy Local Models: concepts/inference-services.md - Troubleshooting: troubleshooting.md - Tutorials: - Overview: tutorials/index.md diff --git a/skills/anonymizer/SKILL.md b/skills/anonymizer/SKILL.md index 863d49d5..ad43017a 100644 --- a/skills/anonymizer/SKILL.md +++ b/skills/anonymizer/SKILL.md @@ -48,7 +48,7 @@ regulatory and business context. - **`risk_tolerance` only applies to Rewrite mode**, not Replace. - **`PrivacyGoal.protect` and `.preserve` must each be 10–1000 chars and at least 3 words.** Be specific (categories, named identifiers, structural facets); avoid generic phrasing like "preserve meaning". - **Validator pool is the only model role with built-in load-spreading.** Set `entity_validator: [a, b, c]` in `models.yaml` if rate limits drop rows. Other roles (rewriter, evaluator, etc.) are single-alias. -- **Self-hosted GLiNER:** When detection must not call `build.nvidia.com` (PHI on-prem, air-gapped, latency), use `tools/inference_service.py` from a **source checkout** to compile the pinned `tools/inference_service_profiles/nvidia-gliner.toml` profile and launch its managed plan. The profile uses vLLM 0.27.1 and the external vLLM Factory project at a pinned source revision. Local serving requires Python 3.12 or later. The tool and runtime are not installed by `pip install nemo-anonymizer`; install the repository's `local-models` dependency group first. Add a provider with `endpoint: http://localhost:8001/v1`, then route `entity_detector` through a `gliner-pii-detector` alias with `provider: local-gliner`. Match the compiled host and port in the provider endpoint. `model_configs` is a **complete** model pool, not an overlay. Copy `src/anonymizer/config/default_model_configs/models.yaml` and change only the detector entry, keeping `gpt-oss-120b` and `nemotron-30b-thinking`. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). +- **Self-hosted GLiNER:** When detection must not call `build.nvidia.com` (PHI on-prem, air-gapped, latency), use `tools/inference_service.py` from a **source checkout** to compile the pinned `tools/inference_service_profiles/nvidia-gliner.toml` profile and launch its managed plan. The profile uses vLLM 0.27.1 and the external vLLM Factory project at a pinned source revision. Local serving requires Python 3.12 or later. Deploy on the GPU host or build `tools/inference_service.Dockerfile` and run the same lifecycle inside a GPU container. The tool and runtime are not installed by `pip install nemo-anonymizer`. Add a provider with `endpoint: http://localhost:8001/v1`, then route `entity_detector` through a `gliner-pii-detector` alias with `provider: local-gliner`. Match the compiled host and port in the provider endpoint. `model_configs` is a **complete** model pool, not an overlay. Copy `src/anonymizer/config/default_model_configs/models.yaml` and change only the detector entry, keeping `gpt-oss-120b` and `nemotron-30b-thinking`. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published deployment guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). - **The evaluation judges use their own model roles** (`entity_coverage_judge`, `detection_validity_judge`, `replace_type_fidelity_judge`, `replace_relational_consistency_judge`, `replace_attribute_fidelity_judge`, `rewrite_judge`), configured in the `evaluate` section of `models.yaml`. They are **not** consumed by `preview()` / `run()`, so a config that anonymizes fine can still fail validation at `evaluate()` if those roles are unset. Defaults ship in `src/anonymizer/config/default_model_configs/evaluate.yaml` (`entity_coverage_judge` defaults to `nemotron-super`). - **Verdict columns are null when the judge was unavailable** — `None` means "unscored", never a pass. `entity_coverage` is a `0–1` float (`1.0` = no missed candidate values or no PII found) or `None`; `missed_entities` lists unique candidate values the anonymizer failed to detect. Replace verdict columns (`type_fidelity_valid`, etc.) are `True` / `False` / `None`. Rewrite `detection_valid` is a `0–1` float fraction (or `None` if unscored). Inspect verdicts per record with `evaluated.display_record(i)`. - **`EvaluateConfig` has one knob today: `compute_detection_validity`** (default `False`). Plain `anonymizer.evaluate(result)` runs entity coverage + the mode's quality judges; pass `EvaluateConfig(compute_detection_validity=True)` only to additionally score detection validity (an internal-facing tag-precision metric). @@ -73,7 +73,7 @@ read `docs/troubleshooting.md` or the - **`anonymizer` not installed:** Tell the user `nemo-anonymizer` is not in this Python environment (requires Python ≥ 3.11). Ask if they want you to install it (`pip install nemo-anonymizer`) or do it themselves. Do not install without permission. - **Model/provider setup:** Plain `Anonymizer()` ships with bundled `models.yaml` and `providers.yaml` (see `src/anonymizer/config/default_model_configs/`). For the default path, confirm `NVIDIA_API_KEY` is set. Pass custom `model_configs` or `model_providers` only for non-default endpoints or model pools. See `docs/concepts/models.md` or the [published models guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/models/). - **LLM calls failing at preview:** Check for a missing or invalid API key, a network problem, or a wrong endpoint URL. See `docs/troubleshooting.md` "Validation passed but `preview` errors at LLM call" or the [published troubleshooting guide](https://nvidia-nemo.github.io/Anonymizer/dev/troubleshooting/). -- **Local / on-prem GLiNER:** Clone the Anonymizer repository, install its `local-models` group, then compile and launch `tools/inference_service_profiles/nvidia-gliner.toml` with `tools/inference_service.py`. Add a provider with the plan's endpoint (normally `http://localhost:8001/v1`) and point `gliner-pii-detector` at `provider: local-gliner`. The vLLM Factory adapter supports DataDesigner's health check, so do not suppress it. Preflight errors about missing aliases usually mean `model_configs` lists only the detector. Include the full default pool. Use the launch receipt to inspect or cancel the managed process. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published inference-services guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). +- **Local / on-prem GLiNER:** Clone the Anonymizer repository, then deploy `tools/inference_service_profiles/nvidia-gliner.toml` on the GPU host or through `tools/inference_service.Dockerfile`. Add a provider with the plan's endpoint (normally `http://localhost:8001/v1`) and point `gliner-pii-detector` at `provider: local-gliner`. The vLLM Factory adapter supports DataDesigner's health check, so do not suppress it. Preflight errors about missing aliases usually mean `model_configs` lists only the detector. Include the full default pool. Use the launch receipt to inspect or cancel the managed process before stopping its container. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published deployment guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). # Output Template diff --git a/tools/inference_service.Dockerfile b/tools/inference_service.Dockerfile new file mode 100644 index 00000000..a127c627 --- /dev/null +++ b/tools/inference_service.Dockerfile @@ -0,0 +1,26 @@ +# syntax=docker/dockerfile:1 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +FROM nvidia/cuda:13.0.2-devel-ubuntu24.04 + +COPY --from=ghcr.io/astral-sh/uv:0.11.31 /uv /uvx /bin/ + +ENV DEBIAN_FRONTEND=noninteractive \ + HF_HOME=/models/huggingface \ + UV_LINK_MODE=copy \ + UV_PROJECT_ENVIRONMENT=/opt/anonymizer/.venv \ + UV_PYTHON_INSTALL_DIR=/opt/uv-python \ + PATH=/opt/anonymizer/.venv/bin:$PATH + +RUN apt-get update \ + && apt-get install --yes --no-install-recommends ca-certificates git \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/anonymizer +COPY . . + +RUN uv python install 3.12 \ + && uv sync --frozen --python 3.12 --no-default-groups --group local-models + +CMD ["sleep", "infinity"] From e6d00c59eb88a45a9a86f896c1e328a7ae51a0a0 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 13 Aug 2026 22:17:06 +0000 Subject: [PATCH 11/28] refactor(dev): close inference service compiler types Signed-off-by: Aaron Gonzales --- ruff.toml | 2 +- tests/tools/test_inference_service.py | 8 + tools/inference_service_compiler/cli.py | 15 +- tools/inference_service_compiler/compiler.py | 145 ++++++++++-------- tools/inference_service_compiler/models.py | 16 +- tools/inference_service_compiler/runtime.py | 31 ++-- .../vllm_factory_adapter.py | 37 +++-- .../vllm_factory_integration.py | 37 +++-- .../vllm_runtime.py | 9 +- 9 files changed, 187 insertions(+), 113 deletions(-) diff --git a/ruff.toml b/ruff.toml index d7b98193..20659382 100644 --- a/ruff.toml +++ b/ruff.toml @@ -34,7 +34,7 @@ fixable = ["ALL"] unfixable = [] [lint.isort] -known-first-party = ["anonymizer"] +known-first-party = ["anonymizer", "inference_service_compiler"] [lint.flake8-tidy-imports] ban-relative-imports = "all" diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index b7d0fd38..f47eec5d 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -128,6 +128,14 @@ def test_factory_detection_is_task_bounded() -> None: ) +def test_factory_plugin_is_closed_at_the_intent_boundary() -> None: + """Profiles cannot select an uncharacterized Factory plugin.""" + models, _compiler, _runtime = modules() + + with pytest.raises(ValidationError): + models.VllmFactoryIntegration.model_validate({"plugin": "unsupported"}) + + def test_factory_detection_requires_a_pin_and_characterized_model() -> None: models, compiler, _runtime = modules() for model_id, revision, message in ( diff --git a/tools/inference_service_compiler/cli.py b/tools/inference_service_compiler/cli.py index 4d613523..cb75c9e7 100644 --- a/tools/inference_service_compiler/cli.py +++ b/tools/inference_service_compiler/cli.py @@ -9,12 +9,13 @@ import sys from collections.abc import Callable from pathlib import Path +from tomllib import TOMLDecodeError from typing import ParamSpec, TypeVar import cyclopts from pydantic import BaseModel, ValidationError -from inference_service_compiler.compiler import CompilationError, compile_intent, load_plan +from inference_service_compiler.compiler import CompilationError, PlanIntegrityError, compile_intent, load_plan from inference_service_compiler.models import LaunchReceipt, SecretEnvironmentVariable from inference_service_compiler.profiles import load_profile from inference_service_compiler.runtime import ( @@ -38,7 +39,17 @@ def command_errors(function: Callable[P, R]) -> Callable[P, R]: def wrapped(*args: P.args, **kwargs: P.kwargs) -> R: try: return function(*args, **kwargs) - except (CompilationError, OSError, RuntimeEffectError, ValidationError, ValueError) as exc: + except ( + CompilationError, + FileNotFoundError, + IsADirectoryError, + PermissionError, + PlanIntegrityError, + RuntimeEffectError, + TOMLDecodeError, + UnicodeDecodeError, + ValidationError, + ) as exc: sys.stderr.write(f"error: {exc}\n") raise SystemExit(125) from exc diff --git a/tools/inference_service_compiler/compiler.py b/tools/inference_service_compiler/compiler.py index 11e86976..dbfda077 100644 --- a/tools/inference_service_compiler/compiler.py +++ b/tools/inference_service_compiler/compiler.py @@ -7,7 +7,8 @@ import hashlib import hmac import json -from typing import Never +from dataclasses import dataclass +from typing import Never, assert_never from pydantic import BaseModel @@ -62,20 +63,30 @@ class PlanIntegrityError(ValueError): """A serialized plan does not match its declared digest.""" +@dataclass(frozen=True, slots=True) +class ServiceCompilation: + """Complete immutable product of selecting one service implementation.""" + + command: CommandSpec + runtime: LocalProcessRuntime + declared_capabilities: tuple[Capability, ...] + compatibility_evidence: tuple[CompatibilityEvidence, ...] + + def compile_intent(intent: InferenceIntent, *, source_revision: str) -> RunPlan: """Compile semantic intent without starting, probing, or allocating anything.""" if not source_revision: raise ValueError("source_revision must not be empty") required = intent.task.required_capabilities() - command, runtime, declared, evidence = _compile_service(intent) + compilation = _compile_service(intent) placement = intent.local endpoint = EndpointContract(host=placement.host, port=placement.port) plan = RunPlan( plan_digest="", intent_digest=digest_model(intent), intent=intent, - command=command, - runtime=runtime, + command=compilation.command, + runtime=compilation.runtime, endpoint=endpoint, readiness=HttpProbe( host=placement.host, @@ -86,8 +97,8 @@ def compile_intent(intent: InferenceIntent, *, source_revision: str) -> RunPlan: ), expected_model=intent.expected_model, required_capabilities=required, - declared_capabilities=declared, - compatibility_evidence=evidence, + declared_capabilities=compilation.declared_capabilities, + compatibility_evidence=compilation.compatibility_evidence, dependencies=_plan_dependencies(intent), source_revision=source_revision, ) @@ -124,78 +135,80 @@ def _canonical_json(value: object) -> bytes: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() -def _compile_service( - intent: InferenceIntent, -) -> tuple[CommandSpec, LocalProcessRuntime, tuple[Capability, ...], tuple[CompatibilityEvidence, ...]]: +def _compile_service(intent: InferenceIntent) -> ServiceCompilation: return _compile_vllm(intent, intent.vllm) def _compile_vllm( intent: InferenceIntent, engine: Vllm, -) -> tuple[CommandSpec, LocalProcessRuntime, tuple[Capability, ...], tuple[CompatibilityEvidence, ...]]: - if isinstance(intent.task, EntityDetection): - factory = engine.factory - if factory is None: - _raise_unsupported_task_engine(intent.task.kind, "vllm") - if intent.model.revision is None: - raise CompilationError( - CompilerDiagnostic( - code="unpinned-model-revision", - message="vLLM Factory entity detection requires a pinned model revision", - details={"model": intent.model.model_id}, +) -> ServiceCompilation: + match intent.task: + case EntityDetection() as task: + factory = engine.factory + if factory is None: + _raise_unsupported_task_engine(task.kind, "vllm") + if intent.model.revision is None: + raise CompilationError( + CompilerDiagnostic( + code="unpinned-model-revision", + message="vLLM Factory entity detection requires a pinned model revision", + details={"model": intent.model.model_id}, + ) ) - ) - if not supports_model(factory.plugin, intent.model.model_id): - raise CompilationError( - CompilerDiagnostic( - code="unsupported-model-engine", - message=( - f"model {intent.model.model_id!r} is not characterized for " - f"vLLM Factory plugin {factory.plugin!r}" - ), - details={ - "model": intent.model.model_id, - "plugin": factory.plugin, - }, + if not supports_model(factory.plugin, intent.model.model_id): + raise CompilationError( + CompilerDiagnostic( + code="unsupported-model-engine", + message=( + f"model {intent.model.model_id!r} is not characterized for " + f"vLLM Factory plugin {factory.plugin!r}" + ), + details={"model": intent.model.model_id, "plugin": factory.plugin}, + ) ) - ) - if intent.model.adapter is not None: - raise CompilationError( - CompilerDiagnostic( - code="unsupported-model-adapter", - message="vLLM Factory entity detection does not support a model adapter", - details={"engine": "vllm", "task": intent.task.kind}, + if intent.model.adapter is not None: + raise CompilationError( + CompilerDiagnostic( + code="unsupported-model-adapter", + message="vLLM Factory entity detection does not support a model adapter", + details={"engine": "vllm", "task": task.kind}, + ) ) + command, runtime = _vllm_command(intent, engine) + return ServiceCompilation( + command=command, + runtime=runtime, + declared_capabilities=task.required_capabilities(), + compatibility_evidence=( + CompatibilityEvidence( + rule="vllm-factory-entity-detection-v1", + outcome="runtime-probe-required", + detail=( + "vLLM Factory supplies model preparation, pooling inference, and IO processing; " + "the Anonymizer adapter preserves dynamic labels, offsets, and scores" + ), + ), + ), ) - command, runtime = _vllm_command(intent, engine) - return ( - command, - runtime, - intent.task.required_capabilities(), - ( - CompatibilityEvidence( - rule="vllm-factory-entity-detection-v1", - outcome="runtime-probe-required", - detail=( - "vLLM Factory supplies model preparation, pooling inference, and IO processing; " - "the Anonymizer adapter preserves dynamic labels, offsets, and scores" + case Generation() as task: + if engine.factory is not None: + _raise_unsupported_task_engine(task.kind, "vllm") + command, runtime = _vllm_command(intent, engine) + return ServiceCompilation( + command=command, + runtime=runtime, + declared_capabilities=("chat-completions",), + compatibility_evidence=( + CompatibilityEvidence( + rule="vllm-openai-compatible-v1", + outcome="characterized", + detail="vLLM exposes chat completions for generation", ), ), - ), - ) - if not isinstance(intent.task, Generation) or engine.factory is not None: - _raise_unsupported_task_engine(intent.task.kind, "vllm") - command, runtime = _vllm_command(intent, engine) - declared = ("chat-completions",) - evidence = ( - CompatibilityEvidence( - rule="vllm-openai-compatible-v1", - outcome="characterized", - detail="vLLM exposes chat completions for generation", - ), - ) - return command, runtime, declared, evidence + ) + case _: + assert_never(intent.task) def _vllm_command( diff --git a/tools/inference_service_compiler/models.py b/tools/inference_service_compiler/models.py index e92de96a..5e23bd32 100644 --- a/tools/inference_service_compiler/models.py +++ b/tools/inference_service_compiler/models.py @@ -5,7 +5,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Annotated, Literal +from typing import Annotated, Literal, assert_never from pydantic import BaseModel, ConfigDict, Field @@ -17,6 +17,16 @@ CANCELLATION_RECEIPT_SCHEMA_VERSION = "inference-service.cancellation-receipt/v1" Capability = Literal["chat-completions", "dynamic-labels", "offsets", "scores"] +FactoryPlugin = Literal["deberta_gliner", "deberta_gliner2"] + + +def parse_factory_plugin(value: str) -> FactoryPlugin: + """Parse the Factory plugin name at an untyped process boundary.""" + match value: + case "deberta_gliner" | "deberta_gliner2": + return value + case _: + raise ValueError(f"unsupported vLLM Factory plugin {value!r}") class FrozenModel(BaseModel): @@ -77,7 +87,7 @@ class HuggingFaceModel(FrozenModel): class VllmFactoryIntegration(FrozenModel): """A supported vLLM Factory structured-prediction plugin.""" - plugin: Literal["deberta_gliner", "deberta_gliner2"] + plugin: FactoryPlugin prepared_model_root: str = Field(default="/tmp/anonymizer-vllm-factory", min_length=1) @@ -184,6 +194,8 @@ def render_environment(self, *, resolve_secrets: Mapping[str, str] | None = None values[name] = resolve_secrets[source] else: raise ValueError(f"secret environment variable {source!r} is not resolved") + case _: + assert_never(variable) return values diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index 30d83d0e..5f4f11f9 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -10,9 +10,10 @@ import subprocess import time from collections.abc import Mapping +from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path -from typing import Literal, cast +from typing import Literal, assert_never, cast import httpx @@ -40,6 +41,14 @@ def __init__(self, diagnostic: RuntimeDiagnostic) -> None: self.diagnostic = diagnostic +@dataclass(frozen=True, slots=True) +class StopOutcome: + """Immutable result of terminating one managed process group.""" + + outcome: Literal["terminated", "forced"] + cleanup_complete: bool + + def probe_endpoint( plan: RunPlan, *, @@ -162,13 +171,13 @@ def cancel_run(launch: LaunchReceipt) -> CancellationReceipt: outcome="already-stopped", cleanup_complete=True, ) - outcome, cleanup_complete = _stop_running_handle(handle, launch.shutdown_timeout_seconds) + stop = _stop_running_handle(handle, launch.shutdown_timeout_seconds) return CancellationReceipt( plan_digest=launch.plan_digest, canceled_at=_now(), handle=handle, - outcome=outcome, - cleanup_complete=cleanup_complete, + outcome=stop.outcome, + cleanup_complete=stop.cleanup_complete, ) @@ -191,18 +200,17 @@ def is_handle_running(handle: LocalProcessHandle) -> bool: def _cleanup_handle(handle: LocalProcessHandle, timeout_seconds: float) -> bool: if not is_handle_running(handle): return True - _outcome, cleanup_complete = _stop_running_handle(handle, timeout_seconds) - return cleanup_complete + return _stop_running_handle(handle, timeout_seconds).cleanup_complete def _stop_running_handle( handle: LocalProcessHandle, timeout_seconds: float, -) -> tuple[Literal["terminated", "forced"], bool]: +) -> StopOutcome: try: os.killpg(handle.process_group_id, signal.SIGTERM) except ProcessLookupError: - return "terminated", True + return StopOutcome(outcome="terminated", cleanup_complete=True) deadline = time.monotonic() + timeout_seconds running = True while time.monotonic() < deadline: @@ -215,10 +223,10 @@ def _stop_running_handle( try: os.killpg(handle.process_group_id, signal.SIGKILL) except ProcessLookupError: - return "forced", True + return StopOutcome(outcome="forced", cleanup_complete=True) running = is_handle_running(handle) outcome = "forced" - return outcome, not running + return StopOutcome(outcome=outcome, cleanup_complete=not running) def wait_for_readiness( @@ -324,7 +332,8 @@ def _probe_task(plan: RunPlan, client: httpx.Client, headers: Mapping[str, str]) if entities and all(isinstance(entity, dict) and "score" in entity for entity in entities): observed.append("scores") return tuple(observed) - raise TypeError(f"unsupported task type {type(plan.intent.task)!r}") + case _: + assert_never(plan.intent.task) def _parse_models(payload: object) -> tuple[str, ...]: diff --git a/tools/inference_service_compiler/vllm_factory_adapter.py b/tools/inference_service_compiler/vllm_factory_adapter.py index ad1cba09..73cde281 100644 --- a/tools/inference_service_compiler/vllm_factory_adapter.py +++ b/tools/inference_service_compiler/vllm_factory_adapter.py @@ -12,13 +12,15 @@ import uuid from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass -from typing import Any, cast +from typing import Any, assert_never, cast + +from inference_service_compiler.models import FactoryPlugin, parse_factory_plugin DEFAULT_CHUNK_LENGTH = 384 DEFAULT_OVERLAP = 128 -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class DetectionRequest: """Validated Anonymizer detector request.""" @@ -31,7 +33,7 @@ class DetectionRequest: flat_ner: bool -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class Entity: """One entity in Anonymizer's detector response shape.""" @@ -62,7 +64,7 @@ async def anonymizer_chat_compatibility( responses = importlib.import_module("starlette.responses") try: detection = parse_detection_request(await request.json()) - plugin = os.environ["ANONYMIZER_VLLM_FACTORY_PLUGIN"] + plugin = parse_factory_plugin(os.environ["ANONYMIZER_VLLM_FACTORY_PLUGIN"]) entities: list[Entity] = [] if detection.labels: chunks = split_text(detection.text, detection.chunk_length, detection.overlap) @@ -183,7 +185,7 @@ async def invoke_pooling( *, handler: Any, model: str, - plugin: str, + plugin: FactoryPlugin, text: str, labels: tuple[str, ...], threshold: float, @@ -196,13 +198,14 @@ async def invoke_pooling( "labels": list(labels), "threshold": threshold, } - if plugin == "deberta_gliner": - data["flat_ner"] = flat_ner - elif plugin == "deberta_gliner2": - data["include_confidence"] = True - data["include_spans"] = True - else: - raise ValueError(f"unsupported vLLM Factory plugin {plugin!r}") + match plugin: + case "deberta_gliner": + data["flat_ner"] = flat_ner + case "deberta_gliner2": + data["include_confidence"] = True + data["include_spans"] = True + case _: + assert_never(plugin) response = await handler(protocol.IOProcessorRequest(model=model, data=data), None) if response.status_code != 200: raise RuntimeError(f"vLLM Factory pooling request returned {response.status_code}") @@ -217,7 +220,7 @@ async def invoke_pooling( def merge_entities( *, - plugin: str, + plugin: FactoryPlugin, chunks: list[tuple[str, int]], results: Sequence[object], flat_ner: bool, @@ -227,7 +230,13 @@ def merge_entities( raise ValueError("vLLM Factory returned an unexpected result count") entities: list[Entity] = [] for (chunk, offset), result in zip(chunks, results, strict=True): - normalized = normalize_gliner(result) if plugin == "deberta_gliner" else normalize_gliner2(result) + match plugin: + case "deberta_gliner": + normalized = normalize_gliner(result) + case "deberta_gliner2": + normalized = normalize_gliner2(result) + case _: + assert_never(plugin) for entity in normalized: if entity.start < 0 or entity.end < entity.start or entity.end > len(chunk): raise ValueError("vLLM Factory returned an invalid entity span") diff --git a/tools/inference_service_compiler/vllm_factory_integration.py b/tools/inference_service_compiler/vllm_factory_integration.py index 6fada61c..69cfc69d 100644 --- a/tools/inference_service_compiler/vllm_factory_integration.py +++ b/tools/inference_service_compiler/vllm_factory_integration.py @@ -9,17 +9,21 @@ import re from contextlib import contextmanager from pathlib import Path -from typing import Any, Iterator +from typing import Any, Iterator, Literal, assert_never + +from inference_service_compiler.models import FactoryPlugin VLLM_FACTORY_SOURCE_REVISION = "7d6ff68ce68f9f7c0a9d72f9645bcf6d335d02f0" VLLM_FACTORY_SOURCE_URL = "https://github.com/latenceainew/vllm-factory.git" VLLM_FACTORY_DEPENDENCY = f"vllm-factory[gliner] @ git+{VLLM_FACTORY_SOURCE_URL}@{VLLM_FACTORY_SOURCE_REVISION}" -PLUGIN_IO_PROCESSORS = { +FactoryIoProcessor = Literal["deberta_gliner_io", "deberta_gliner2_io"] + +PLUGIN_IO_PROCESSORS: dict[FactoryPlugin, FactoryIoProcessor] = { "deberta_gliner": "deberta_gliner_io", "deberta_gliner2": "deberta_gliner2_io", } -CHARACTERIZED_MODELS = { +CHARACTERIZED_MODELS: dict[FactoryPlugin, frozenset[str]] = { "deberta_gliner": frozenset({"nvidia/gliner-pii"}), "deberta_gliner2": frozenset({"fastino/gliner2-privacy-filter-PII-multi"}), } @@ -29,12 +33,10 @@ def prepare_model( *, model_id: str, revision: str | None, - plugin: str, + plugin: FactoryPlugin, prepared_model_root: str, ) -> str: """Prepare one pinned model through vLLM Factory's Python API.""" - if plugin not in PLUGIN_IO_PROCESSORS: - raise ValueError(f"unsupported vLLM Factory plugin {plugin!r}") if revision is None: raise ValueError("vLLM Factory models require a pinned Hugging Face revision") @@ -70,20 +72,27 @@ def prepare_model( return str(output) -def io_processor_for(plugin: str) -> str: +def io_processor_for(plugin: FactoryPlugin) -> FactoryIoProcessor: """Resolve the vLLM IOProcessor entry point for a supported factory plugin.""" - try: - return PLUGIN_IO_PROCESSORS[plugin] - except KeyError as exc: - raise ValueError(f"unsupported vLLM Factory plugin {plugin!r}") from exc + match plugin: + case "deberta_gliner": + return "deberta_gliner_io" + case "deberta_gliner2": + return "deberta_gliner2_io" + case _: + assert_never(plugin) -def supports_model(plugin: str, model_id: str) -> bool: +def supports_model(plugin: FactoryPlugin, model_id: str) -> bool: """Return whether this source revision was characterized for the pair.""" - return model_id in CHARACTERIZED_MODELS.get(plugin, ()) + match plugin: + case "deberta_gliner" | "deberta_gliner2": + return model_id in CHARACTERIZED_MODELS[plugin] + case _: + assert_never(plugin) -def _prepared_model_path(*, root: Path, model_id: str, revision: str, plugin: str) -> Path: +def _prepared_model_path(*, root: Path, model_id: str, revision: str, plugin: FactoryPlugin) -> Path: safe_model = re.sub(r"[^A-Za-z0-9_.-]+", "--", model_id).strip("-") return root / safe_model / revision / plugin diff --git a/tools/inference_service_compiler/vllm_runtime.py b/tools/inference_service_compiler/vllm_runtime.py index 1b73107f..79346180 100644 --- a/tools/inference_service_compiler/vllm_runtime.py +++ b/tools/inference_service_compiler/vllm_runtime.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import Literal +from inference_service_compiler.models import FactoryPlugin, parse_factory_plugin from inference_service_compiler.vllm_factory_integration import ( io_processor_for, prepare_model, @@ -25,7 +26,7 @@ MINIMUM_VLLM_PYTHON = (3, 12) -@dataclass(frozen=True) +@dataclass(frozen=True, slots=True) class VllmServerParameters: """Bounded settings accepted by the source-owned vLLM server process.""" @@ -47,7 +48,7 @@ class VllmServerParameters: enable_mamba_cache_stochastic_rounding: bool = False mamba_cache_philox_rounds: int = 0 lora_module: str | None = None - vllm_factory_plugin: str | None = None + vllm_factory_plugin: FactoryPlugin | None = None prepared_model_root: str = "/tmp/anonymizer-vllm-factory" @@ -104,7 +105,9 @@ def parse_server_parameters(argv: Sequence[str]) -> VllmServerParameters: enable_mamba_cache_stochastic_rounding=parsed.enable_mamba_cache_stochastic_rounding, mamba_cache_philox_rounds=parsed.mamba_cache_philox_rounds, lora_module=parsed.lora_modules, - vllm_factory_plugin=parsed.vllm_factory_plugin, + vllm_factory_plugin=( + parse_factory_plugin(parsed.vllm_factory_plugin) if parsed.vllm_factory_plugin is not None else None + ), prepared_model_root=parsed.prepared_model_root, ) From f13ae2d20b06856b782e2d21f07cc472555b5cdc Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 13 Aug 2026 22:23:32 +0000 Subject: [PATCH 12/28] test(dev): type inference service tool imports Signed-off-by: Aaron Gonzales --- pyproject.toml | 3 +- tests/tools/test_inference_service.py | 52 ++++++++++++-------- tests/tools/test_vllm_factory.py | 21 +++----- tests/tools/test_vllm_factory_adapter.py | 15 ++---- tests/tools/test_vllm_factory_integration.py | 14 +----- tools/inference_service_compiler/cli.py | 1 + 6 files changed, 48 insertions(+), 58 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5d4f4475..cf999ea7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,7 @@ constraint-dependencies = [ [tool.pytest.ini_options] testpaths = ["tests"] +pythonpath = ["tools"] markers = [ "e2e: end-to-end tests", "integration: integration tests", @@ -99,7 +100,7 @@ include = ["src", "tests", "tests_e2e", "docs", "scripts", "tools"] [tool.ty.environment] python-version = "3.11" -root = ["src", ".", "tools/measurement"] +root = ["src", ".", "tools", "tools/measurement"] [tool.coverage.run] omit = [] diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index f47eec5d..04d944db 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -4,47 +4,43 @@ from __future__ import annotations -import importlib import json import stat -import sys from pathlib import Path -from types import ModuleType -from typing import Any from unittest import mock import httpx import pytest from pydantic import ValidationError +from inference_service_compiler import cli, compiler, models, runtime +from inference_service_compiler.profiles import load_profile + ROOT = Path(__file__).resolve().parents[2] TOOLS = ROOT / "tools" PROFILES = TOOLS / "inference_service_profiles" CLI = TOOLS / "inference_service.py" -def modules() -> tuple[ModuleType, ModuleType, ModuleType]: - sys.path.insert(0, str(TOOLS)) - try: - return ( - importlib.import_module("inference_service_compiler.models"), - importlib.import_module("inference_service_compiler.compiler"), - importlib.import_module("inference_service_compiler.runtime"), - ) - finally: - sys.path.pop(0) +def modules(): + """Return directly imported production modules for concise existing tests.""" + return models, compiler, runtime -def generation(models: ModuleType, **vllm: object) -> Any: +def generation(_models: object = None, **vllm: object) -> models.InferenceIntent: return models.InferenceIntent( task=models.Generation(), model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b", revision="abc"), - vllm=models.Vllm(**vllm), + vllm=models.Vllm.model_validate(vllm), local=models.LocalProcess(port=8000, startup_timeout_seconds=3, shutdown_timeout_seconds=0.01), ) -def launch_receipt(models: ModuleType, plan: Any, handle: Any) -> Any: +def launch_receipt( + _models: object, + plan: models.RunPlan, + handle: models.LocalProcessHandle, +) -> models.LaunchReceipt: return models.LaunchReceipt( plan_digest=plan.plan_digest, launched_at="2026-08-07T00:00:00+00:00", @@ -68,8 +64,6 @@ def test_cli_remains_a_directly_executable_source_entrypoint() -> None: def test_all_shipped_profiles_compile() -> None: models, compiler, _runtime = modules() - load_profile = importlib.import_module("inference_service_compiler.profiles").load_profile - plans = [ compiler.compile_intent(load_profile(path), source_revision="test") for path in sorted(PROFILES.glob("*.toml")) ] @@ -79,7 +73,6 @@ def test_all_shipped_profiles_compile() -> None: def test_compile_command_writes_a_digest_verified_plan(tmp_path: Path) -> None: _models, compiler, _runtime = modules() - cli = importlib.import_module("inference_service_compiler.cli") output = tmp_path / "plan.json" with pytest.raises(SystemExit) as exc_info: cli.app( @@ -98,6 +91,25 @@ def test_compile_command_writes_a_digest_verified_plan(tmp_path: Path) -> None: assert plan.expected_model == "anonymizer-local" +def test_compile_command_translates_non_directory_profile_paths(tmp_path: Path) -> None: + """Filesystem path-shape errors retain the documented bad-input exit.""" + not_a_directory = tmp_path / "profile-file" + not_a_directory.write_text("not a directory", encoding="utf-8") + + with pytest.raises(SystemExit) as exc_info: + cli.app( + [ + "compile", + "--profile", + str(not_a_directory / "profile.toml"), + "--source-revision", + "test", + ] + ) + + assert exc_info.value.code == 125 + + def test_generation_argv_keeps_local_vllm_controls_and_omits_defaults() -> None: models, compiler, _runtime = modules() plan = compiler.compile_intent( diff --git a/tests/tools/test_vllm_factory.py b/tests/tools/test_vllm_factory.py index 8995da27..40a66b4b 100644 --- a/tests/tools/test_vllm_factory.py +++ b/tests/tools/test_vllm_factory.py @@ -6,17 +6,23 @@ import importlib import os -import sys import tomllib from pathlib import Path from unittest import mock import pytest +from inference_service_compiler import vllm_runtime as factory + TOOLS_ROOT = Path(__file__).resolve().parents[2] / "tools" REPO_ROOT = TOOLS_ROOT.parent +def load_factory_module(): + """Compatibility alias for the directly imported production module.""" + return factory + + def test_local_models_group_pins_vllm_and_external_factory_source() -> None: """The runtime pins vLLM and the reviewed external factory source revision.""" project = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) @@ -34,19 +40,8 @@ def test_local_models_group_pins_vllm_and_external_factory_source() -> None: ] -def load_factory_module(): - """Load the source-tree runtime without packaging it.""" - sys.path.insert(0, str(TOOLS_ROOT)) - try: - return importlib.import_module("inference_service_compiler.vllm_runtime") - finally: - sys.path.pop(0) - - def test_parse_server_parameters_accepts_only_the_compiler_contract() -> None: """The process entry point accepts the bounded arguments emitted by the compiler.""" - factory = load_factory_module() - parameters = factory.parse_server_parameters( [ "TinyLlama/TinyLlama-1.1B-Chat-v1.0", @@ -101,7 +96,6 @@ def test_parse_server_parameters_accepts_only_the_compiler_contract() -> None: def test_factory_constructs_vllm_frontend_and_async_engine_arguments() -> None: """The local service is built from vLLM Python config objects.""" pytest.importorskip("vllm") - factory = load_factory_module() parameters = factory.VllmServerParameters( model="TinyLlama/TinyLlama-1.1B-Chat-v1.0", host="127.0.0.1", @@ -149,7 +143,6 @@ def test_factory_constructs_vllm_frontend_and_async_engine_arguments() -> None: def test_factory_constructs_pooling_server_for_external_gliner_plugin() -> None: """Factory-backed detection uses vLLM pooling and the project's IOProcessor.""" pytest.importorskip("vllm") - factory = load_factory_module() parameters = factory.VllmServerParameters( model="/tmp/prepared-gliner", host="127.0.0.1", diff --git a/tests/tools/test_vllm_factory_adapter.py b/tests/tools/test_vllm_factory_adapter.py index be91ac65..33d72815 100644 --- a/tests/tools/test_vllm_factory_adapter.py +++ b/tests/tools/test_vllm_factory_adapter.py @@ -4,26 +4,20 @@ from __future__ import annotations -import importlib -import sys from pathlib import Path +from inference_service_compiler import vllm_factory_adapter as adapter + TOOLS_ROOT = Path(__file__).resolve().parents[2] / "tools" def load_adapter(): - """Load the source-tree adapter without installing it as a package.""" - sys.path.insert(0, str(TOOLS_ROOT)) - try: - return importlib.import_module("inference_service_compiler.vllm_factory_adapter") - finally: - sys.path.pop(0) + """Compatibility alias for the directly imported production module.""" + return adapter def test_parse_detection_request_preserves_anonymizer_options() -> None: """The adapter accepts the detector extras emitted by DataDesigner.""" - adapter = load_adapter() - request = adapter.parse_detection_request( { "model": "nvidia/gliner-pii", @@ -46,7 +40,6 @@ def test_parse_detection_request_preserves_anonymizer_options() -> None: def test_parse_detection_request_accepts_label_free_health_check() -> None: """DataDesigner's generic model health check receives a valid empty result.""" - adapter = load_adapter() request = adapter.parse_detection_request( { diff --git a/tests/tools/test_vllm_factory_integration.py b/tests/tools/test_vllm_factory_integration.py index 0ecc6787..732f9077 100644 --- a/tests/tools/test_vllm_factory_integration.py +++ b/tests/tools/test_vllm_factory_integration.py @@ -4,27 +4,17 @@ from __future__ import annotations -import importlib -import sys from pathlib import Path from types import SimpleNamespace from unittest import mock -TOOLS_ROOT = Path(__file__).resolve().parents[2] / "tools" - +from inference_service_compiler import vllm_factory_integration as integration -def load_integration(): - """Load the source-tree integration without installing it as a package.""" - sys.path.insert(0, str(TOOLS_ROOT)) - try: - return importlib.import_module("inference_service_compiler.vllm_factory_integration") - finally: - sys.path.pop(0) +TOOLS_ROOT = Path(__file__).resolve().parents[2] / "tools" def test_prepare_model_injects_pinned_revision_into_upstream_python_api(tmp_path: Path) -> None: """Anonymizer closes vLLM Factory's missing Hugging Face revision argument.""" - integration = load_integration() download = mock.Mock(return_value="/cache/file") list_files = mock.Mock(return_value=[]) tokenizer = mock.Mock(return_value=mock.Mock()) diff --git a/tools/inference_service_compiler/cli.py b/tools/inference_service_compiler/cli.py index cb75c9e7..90048694 100644 --- a/tools/inference_service_compiler/cli.py +++ b/tools/inference_service_compiler/cli.py @@ -43,6 +43,7 @@ def wrapped(*args: P.args, **kwargs: P.kwargs) -> R: CompilationError, FileNotFoundError, IsADirectoryError, + NotADirectoryError, PermissionError, PlanIntegrityError, RuntimeEffectError, From 65f793a7a3bb529cc24df6a0cc6dca7b903483d6 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 13 Aug 2026 22:30:48 +0000 Subject: [PATCH 13/28] refactor(dev): close inference service boundaries Signed-off-by: Aaron Gonzales --- tests/tools/test_inference_service.py | 2 + tests/tools/test_vllm_factory_adapter.py | 8 ++-- tests/tools/test_vllm_factory_integration.py | 11 +++++ tools/inference_service_compiler/cli.py | 9 +--- tools/inference_service_compiler/compiler.py | 12 ++--- tools/inference_service_compiler/models.py | 36 +++++++++++---- tools/inference_service_compiler/runtime.py | 38 +++------------- .../vllm_factory_adapter.py | 30 ++++++++----- .../vllm_factory_integration.py | 44 ++++++++++--------- 9 files changed, 100 insertions(+), 90 deletions(-) diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index 04d944db..36031d55 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -122,7 +122,9 @@ def test_generation_argv_keeps_local_vllm_controls_and_omits_defaults() -> None: argv.index("--tensor-parallel-size") : argv.index("--tensor-parallel-size") + 2 ] assert "--enforce-eager" in argv and "--enable-prefix-caching" not in argv + assert plan.command.secret_sources == ("LOCAL_KEY",) assert plan.command.render_environment() == {"VLLM_API_KEY": ""} + assert plan.command.resolve_environment({"LOCAL_KEY": "secret-value"}) == {"VLLM_API_KEY": "secret-value"} def test_factory_detection_is_task_bounded() -> None: diff --git a/tests/tools/test_vllm_factory_adapter.py b/tests/tools/test_vllm_factory_adapter.py index 33d72815..fe098168 100644 --- a/tests/tools/test_vllm_factory_adapter.py +++ b/tests/tools/test_vllm_factory_adapter.py @@ -54,7 +54,7 @@ def test_parse_detection_request_accepts_label_free_health_check() -> None: def test_merge_gliner_entities_restores_offsets_and_deduplicates_overlap() -> None: """Chunk-relative factory spans become stable document offsets.""" adapter = load_adapter() - chunks = [("Alice met Bob", 0), ("Bob at NVIDIA", 10)] + chunks = [adapter.TextChunk("Alice met Bob", 0), adapter.TextChunk("Bob at NVIDIA", 10)] results = [ [ {"text": "Alice", "label": "person", "start": 0, "end": 5, "score": 0.9}, @@ -86,7 +86,7 @@ def test_merge_gliner2_entities_normalizes_confidence_and_spans() -> None: entities = adapter.merge_entities( plugin="deberta_gliner2", - chunks=[("Email alice@example.com", 0)], + chunks=[adapter.TextChunk("Email alice@example.com", 0)], results=[ { "entities": { @@ -120,6 +120,6 @@ def test_split_text_matches_native_character_overlap_contract() -> None: adapter = load_adapter() assert adapter.split_text("abcdefghij", chunk_length=6, overlap=2) == [ - ("abcdef", 0), - ("efghij", 4), + adapter.TextChunk("abcdef", 0), + adapter.TextChunk("efghij", 4), ] diff --git a/tests/tools/test_vllm_factory_integration.py b/tests/tools/test_vllm_factory_integration.py index 732f9077..2866d5e6 100644 --- a/tests/tools/test_vllm_factory_integration.py +++ b/tests/tools/test_vllm_factory_integration.py @@ -13,6 +13,17 @@ TOOLS_ROOT = Path(__file__).resolve().parents[2] / "tools" +def test_factory_plugin_spec_is_immutable_and_complete() -> None: + """Each supported plugin resolves all of its metadata through one product.""" + gliner = integration.factory_plugin_spec("deberta_gliner") + gliner2 = integration.factory_plugin_spec("deberta_gliner2") + + assert gliner.io_processor == "deberta_gliner_io" + assert gliner.characterized_models == frozenset({"nvidia/gliner-pii"}) + assert gliner2.io_processor == "deberta_gliner2_io" + assert gliner2.characterized_models == frozenset({"fastino/gliner2-privacy-filter-PII-multi"}) + + def test_prepare_model_injects_pinned_revision_into_upstream_python_api(tmp_path: Path) -> None: """Anonymizer closes vLLM Factory's missing Hugging Face revision argument.""" download = mock.Mock(return_value="/cache/file") diff --git a/tools/inference_service_compiler/cli.py b/tools/inference_service_compiler/cli.py index 90048694..6ef70666 100644 --- a/tools/inference_service_compiler/cli.py +++ b/tools/inference_service_compiler/cli.py @@ -16,7 +16,7 @@ from pydantic import BaseModel, ValidationError from inference_service_compiler.compiler import CompilationError, PlanIntegrityError, compile_intent, load_plan -from inference_service_compiler.models import LaunchReceipt, SecretEnvironmentVariable +from inference_service_compiler.models import LaunchReceipt from inference_service_compiler.profiles import load_profile from inference_service_compiler.runtime import ( RuntimeEffectError, @@ -81,12 +81,7 @@ def launch( ) -> None: """Launch a compiled plan and write its reconnectable handle receipt.""" parsed = load_plan(plan.read_text(encoding="utf-8")) - required_secrets = { - variable.source_environment_variable - for variable in parsed.command.environment - if isinstance(variable, SecretEnvironmentVariable) - } - secret_values = {name: os.environ[name] for name in required_secrets if name in os.environ} + secret_values = {name: os.environ[name] for name in parsed.command.secret_sources if name in os.environ} write_json( launch_plan(parsed, secret_values=secret_values, log_directory=log_directory), output, diff --git a/tools/inference_service_compiler/compiler.py b/tools/inference_service_compiler/compiler.py index dbfda077..a02b4b91 100644 --- a/tools/inference_service_compiler/compiler.py +++ b/tools/inference_service_compiler/compiler.py @@ -78,7 +78,7 @@ def compile_intent(intent: InferenceIntent, *, source_revision: str) -> RunPlan: if not source_revision: raise ValueError("source_revision must not be empty") required = intent.task.required_capabilities() - compilation = _compile_service(intent) + compilation = _compile_vllm(intent) placement = intent.local endpoint = EndpointContract(host=placement.host, port=placement.port) plan = RunPlan( @@ -135,14 +135,8 @@ def _canonical_json(value: object) -> bytes: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() -def _compile_service(intent: InferenceIntent) -> ServiceCompilation: - return _compile_vllm(intent, intent.vllm) - - -def _compile_vllm( - intent: InferenceIntent, - engine: Vllm, -) -> ServiceCompilation: +def _compile_vllm(intent: InferenceIntent) -> ServiceCompilation: + engine = intent.vllm match intent.task: case EntityDetection() as task: factory = engine.factory diff --git a/tools/inference_service_compiler/models.py b/tools/inference_service_compiler/models.py index 5e23bd32..39dcb119 100644 --- a/tools/inference_service_compiler/models.py +++ b/tools/inference_service_compiler/models.py @@ -180,24 +180,44 @@ def render_argv(self) -> tuple[str, ...]: """Render the complete non-secret process argument vector.""" return tuple(argument.value for argument in self.argv) - def render_environment(self, *, resolve_secrets: Mapping[str, str] | None = None) -> dict[str, str]: - """Render environment values with redacted or explicitly supplied secrets.""" + @property + def secret_sources(self) -> tuple[str, ...]: + """Return the secret-source names this command owns.""" + return tuple( + variable.source_environment_variable + for variable in self.environment + if isinstance(variable, SecretEnvironmentVariable) + ) + + def render_environment(self) -> dict[str, str]: + """Render an inspectable environment with every secret source redacted.""" values: dict[str, str] = {} for variable in self.environment: match variable: case EnvironmentVariable(name=name, value=value): values[name] = value case SecretEnvironmentVariable(name=name, source_environment_variable=source): - if resolve_secrets is None: - values[name] = f"" - elif source in resolve_secrets: - values[name] = resolve_secrets[source] - else: - raise ValueError(f"secret environment variable {source!r} is not resolved") + values[name] = f"" case _: assert_never(variable) return values + def resolve_environment(self, secret_values: Mapping[str, str]) -> dict[str, str]: + """Strictly resolve this command's owned secret sources for execution.""" + resolved = self.render_environment() + for variable in self.environment: + match variable: + case EnvironmentVariable(): + pass + case SecretEnvironmentVariable(name=name, source_environment_variable=source): + try: + resolved[name] = secret_values[source] + except KeyError as exc: + raise ValueError(f"secret environment variable {source!r} is not resolved") from exc + case _: + assert_never(variable) + return resolved + class EndpointContract(FrozenModel): """Direct OpenAI-compatible endpoint produced by a run.""" diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index 5f4f11f9..1a50fd24 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -28,7 +28,6 @@ LocalProcessHandle, RunPlan, RuntimeDiagnostic, - SecretEnvironmentVariable, StatusReceipt, ) @@ -94,13 +93,16 @@ def launch_plan( ) -> LaunchReceipt: """Launch a verified plan and return reconnectable identity plus readiness evidence.""" verify_plan(plan) - resolved_secrets = _resolve_secrets(plan, secret_values) + try: + command_environment = plan.command.resolve_environment(secret_values) + except ValueError as exc: + raise RuntimeEffectError(RuntimeDiagnostic(code="missing-secret", message=str(exc))) from exc argv = plan.command.render_argv() environment = os.environ.copy() - environment.update(plan.command.render_environment(resolve_secrets=resolved_secrets)) + environment.update(command_environment) log_directory.mkdir(parents=True, exist_ok=True) - handle = _launch_handle(plan, argv, environment, log_directory) - probe = _probe_or_cleanup(plan, handle, resolved_secrets) + handle = _launch_process(plan, argv, environment, log_directory) + probe = _probe_or_cleanup(plan, handle, secret_values) return LaunchReceipt( plan_digest=plan.plan_digest, launched_at=_now(), @@ -110,15 +112,6 @@ def launch_plan( ) -def _launch_handle( - plan: RunPlan, - argv: tuple[str, ...], - environment: dict[str, str], - log_directory: Path, -) -> LocalProcessHandle: - return _launch_process(plan, argv, environment, log_directory) - - def _probe_or_cleanup( plan: RunPlan, handle: LocalProcessHandle, @@ -351,23 +344,6 @@ def _parse_models(payload: object) -> tuple[str, ...]: return tuple(models) -def _resolve_secrets(plan: RunPlan, values: dict[str, str]) -> dict[str, str]: - required = { - variable.source_environment_variable - for variable in plan.command.environment - if isinstance(variable, SecretEnvironmentVariable) - } - missing = sorted(name for name in required if name not in values) - if missing: - raise RuntimeEffectError( - RuntimeDiagnostic( - code="missing-secret", - message=f"secret environment variable {missing[0]!r} is not resolved", - ) - ) - return {name: values[name] for name in required} - - def _probe_headers(plan: RunPlan, secret_values: Mapping[str, str]) -> dict[str, str]: source = plan.readiness.bearer_token_environment_variable if source is None: diff --git a/tools/inference_service_compiler/vllm_factory_adapter.py b/tools/inference_service_compiler/vllm_factory_adapter.py index 73cde281..a4963d07 100644 --- a/tools/inference_service_compiler/vllm_factory_adapter.py +++ b/tools/inference_service_compiler/vllm_factory_adapter.py @@ -53,6 +53,14 @@ def as_dict(self) -> dict[str, str | int | float]: } +@dataclass(frozen=True, slots=True) +class TextChunk: + """One text segment and its original document offset.""" + + text: str + offset: int + + async def anonymizer_chat_compatibility( request: Any, call_next: Callable[[Any], Awaitable[Any]], @@ -77,12 +85,12 @@ async def anonymizer_chat_compatibility( handler=handler, model=detection.model, plugin=plugin, - text=chunk, + text=chunk.text, labels=detection.labels, threshold=detection.threshold, flat_ner=detection.flat_ner, ) - for chunk, _offset in chunks + for chunk in chunks ) ) entities = merge_entities( @@ -167,14 +175,14 @@ def extract_text(messages: object) -> str: raise ValueError("message content must be a string or list") -def split_text(text: str, chunk_length: int, overlap: int) -> list[tuple[str, int]]: +def split_text(text: str, chunk_length: int, overlap: int) -> list[TextChunk]: """Split text with the same character-offset contract as the native runtime.""" if not text: - return [("", 0)] - chunks: list[tuple[str, int]] = [] + return [TextChunk("", 0)] + chunks: list[TextChunk] = [] start = 0 while start < len(text): - chunks.append((text[start : start + chunk_length], start)) + chunks.append(TextChunk(text[start : start + chunk_length], start)) if start + chunk_length >= len(text): break start += chunk_length - overlap @@ -221,7 +229,7 @@ async def invoke_pooling( def merge_entities( *, plugin: FactoryPlugin, - chunks: list[tuple[str, int]], + chunks: Sequence[TextChunk], results: Sequence[object], flat_ner: bool, ) -> list[Entity]: @@ -229,7 +237,7 @@ def merge_entities( if len(chunks) != len(results): raise ValueError("vLLM Factory returned an unexpected result count") entities: list[Entity] = [] - for (chunk, offset), result in zip(chunks, results, strict=True): + for chunk, result in zip(chunks, results, strict=True): match plugin: case "deberta_gliner": normalized = normalize_gliner(result) @@ -238,14 +246,14 @@ def merge_entities( case _: assert_never(plugin) for entity in normalized: - if entity.start < 0 or entity.end < entity.start or entity.end > len(chunk): + if entity.start < 0 or entity.end < entity.start or entity.end > len(chunk.text): raise ValueError("vLLM Factory returned an invalid entity span") entities.append( Entity( text=entity.text, label=entity.label, - start=entity.start + offset, - end=entity.end + offset, + start=entity.start + chunk.offset, + end=entity.end + chunk.offset, score=entity.score, ) ) diff --git a/tools/inference_service_compiler/vllm_factory_integration.py b/tools/inference_service_compiler/vllm_factory_integration.py index 69cfc69d..4ee28692 100644 --- a/tools/inference_service_compiler/vllm_factory_integration.py +++ b/tools/inference_service_compiler/vllm_factory_integration.py @@ -8,6 +8,7 @@ import json import re from contextlib import contextmanager +from dataclasses import dataclass from pathlib import Path from typing import Any, Iterator, Literal, assert_never @@ -19,14 +20,27 @@ FactoryIoProcessor = Literal["deberta_gliner_io", "deberta_gliner2_io"] -PLUGIN_IO_PROCESSORS: dict[FactoryPlugin, FactoryIoProcessor] = { - "deberta_gliner": "deberta_gliner_io", - "deberta_gliner2": "deberta_gliner2_io", -} -CHARACTERIZED_MODELS: dict[FactoryPlugin, frozenset[str]] = { - "deberta_gliner": frozenset({"nvidia/gliner-pii"}), - "deberta_gliner2": frozenset({"fastino/gliner2-privacy-filter-PII-multi"}), -} + +@dataclass(frozen=True, slots=True) +class FactoryPluginSpec: + """Immutable, characterized contract for one supported Factory plugin.""" + + io_processor: FactoryIoProcessor + characterized_models: frozenset[str] + + +def factory_plugin_spec(plugin: FactoryPlugin) -> FactoryPluginSpec: + """Return the exhaustive immutable specification for a Factory plugin.""" + match plugin: + case "deberta_gliner": + return FactoryPluginSpec("deberta_gliner_io", frozenset({"nvidia/gliner-pii"})) + case "deberta_gliner2": + return FactoryPluginSpec( + "deberta_gliner2_io", + frozenset({"fastino/gliner2-privacy-filter-PII-multi"}), + ) + case _: + assert_never(plugin) def prepare_model( @@ -74,22 +88,12 @@ def prepare_model( def io_processor_for(plugin: FactoryPlugin) -> FactoryIoProcessor: """Resolve the vLLM IOProcessor entry point for a supported factory plugin.""" - match plugin: - case "deberta_gliner": - return "deberta_gliner_io" - case "deberta_gliner2": - return "deberta_gliner2_io" - case _: - assert_never(plugin) + return factory_plugin_spec(plugin).io_processor def supports_model(plugin: FactoryPlugin, model_id: str) -> bool: """Return whether this source revision was characterized for the pair.""" - match plugin: - case "deberta_gliner" | "deberta_gliner2": - return model_id in CHARACTERIZED_MODELS[plugin] - case _: - assert_never(plugin) + return model_id in factory_plugin_spec(plugin).characterized_models def _prepared_model_path(*, root: Path, model_id: str, revision: str, plugin: FactoryPlugin) -> Path: From 7e24ff76a280d821095186b60aab87ef6dff8413 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 13 Aug 2026 22:43:45 +0000 Subject: [PATCH 14/28] fix(dev): preserve inference service boundaries Signed-off-by: Aaron Gonzales --- tests/tools/test_inference_service.py | 71 ++++---- tests/tools/test_vllm_factory.py | 12 -- tests/tools/test_vllm_factory_adapter.py | 14 -- tests/tools/test_vllm_factory_integration.py | 2 - tools/inference_service_compiler/compiler.py | 179 +++++++++++-------- tools/inference_service_compiler/runtime.py | 23 ++- 6 files changed, 142 insertions(+), 159 deletions(-) diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index 36031d55..2870226a 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -22,12 +22,7 @@ CLI = TOOLS / "inference_service.py" -def modules(): - """Return directly imported production modules for concise existing tests.""" - return models, compiler, runtime - - -def generation(_models: object = None, **vllm: object) -> models.InferenceIntent: +def generation(**vllm: object) -> models.InferenceIntent: return models.InferenceIntent( task=models.Generation(), model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b", revision="abc"), @@ -37,7 +32,6 @@ def generation(_models: object = None, **vllm: object) -> models.InferenceIntent def launch_receipt( - _models: object, plan: models.RunPlan, handle: models.LocalProcessHandle, ) -> models.LaunchReceipt: @@ -63,7 +57,6 @@ def test_cli_remains_a_directly_executable_source_entrypoint() -> None: def test_all_shipped_profiles_compile() -> None: - models, compiler, _runtime = modules() plans = [ compiler.compile_intent(load_profile(path), source_revision="test") for path in sorted(PROFILES.glob("*.toml")) ] @@ -72,7 +65,6 @@ def test_all_shipped_profiles_compile() -> None: def test_compile_command_writes_a_digest_verified_plan(tmp_path: Path) -> None: - _models, compiler, _runtime = modules() output = tmp_path / "plan.json" with pytest.raises(SystemExit) as exc_info: cli.app( @@ -110,10 +102,25 @@ def test_compile_command_translates_non_directory_profile_paths(tmp_path: Path) assert exc_info.value.code == 125 +def test_compile_command_translates_empty_source_revision() -> None: + """Known compiler input errors retain the documented bad-input exit.""" + with pytest.raises(SystemExit) as exc_info: + cli.app( + [ + "compile", + "--profile", + str(PROFILES / "vllm-local.toml"), + "--source-revision", + "", + ] + ) + + assert exc_info.value.code == 125 + + def test_generation_argv_keeps_local_vllm_controls_and_omits_defaults() -> None: - models, compiler, _runtime = modules() plan = compiler.compile_intent( - generation(models, api_key_env="LOCAL_KEY", tensor_parallel_size=2, max_model_len=4096, eager=True), + generation(api_key_env="LOCAL_KEY", tensor_parallel_size=2, max_model_len=4096, eager=True), source_revision="test", ) argv = plan.command.render_argv() @@ -128,7 +135,6 @@ def test_generation_argv_keeps_local_vllm_controls_and_omits_defaults() -> None: def test_factory_detection_is_task_bounded() -> None: - models, compiler, _runtime = modules() valid = models.InferenceIntent( task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), model=models.HuggingFaceModel(model_id="nvidia/gliner-pii", revision="abc"), @@ -138,20 +144,17 @@ def test_factory_detection_is_task_bounded() -> None: assert "--vllm-factory-plugin" in compiler.compile_intent(valid, source_revision="test").command.render_argv() with pytest.raises(compiler.CompilationError, match="does not support"): compiler.compile_intent( - generation(models, factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), source_revision="test" + generation(factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), source_revision="test" ) def test_factory_plugin_is_closed_at_the_intent_boundary() -> None: """Profiles cannot select an uncharacterized Factory plugin.""" - models, _compiler, _runtime = modules() - with pytest.raises(ValidationError): models.VllmFactoryIntegration.model_validate({"plugin": "unsupported"}) def test_factory_detection_requires_a_pin_and_characterized_model() -> None: - models, compiler, _runtime = modules() for model_id, revision, message in ( ("nvidia/gliner-pii", None, "pinned model revision"), ("unknown/model", "abc", "not characterized"), @@ -167,7 +170,6 @@ def test_factory_detection_requires_a_pin_and_characterized_model() -> None: def test_removed_domains_are_invalid_profile_fields() -> None: - models, _compiler, _runtime = modules() with pytest.raises(ValidationError): models.InferenceIntent.model_validate( { @@ -182,8 +184,7 @@ def test_removed_domains_are_invalid_profile_fields() -> None: def test_plan_digest_detects_transport_mutation() -> None: - models, compiler, _runtime = modules() - plan = compiler.compile_intent(generation(models), source_revision="test") + plan = compiler.compile_intent(generation(), source_revision="test") changed = json.loads(plan.model_dump_json()) changed["endpoint"]["port"] = 9000 with pytest.raises(compiler.PlanIntegrityError, match="plan digest mismatch"): @@ -191,7 +192,6 @@ def test_plan_digest_detects_transport_mutation() -> None: def test_lora_is_rendered_as_a_model_artifact() -> None: - models, compiler, _runtime = modules() intent = models.InferenceIntent( task=models.Generation(), model=models.HuggingFaceModel( @@ -206,8 +206,7 @@ def test_lora_is_rendered_as_a_model_artifact() -> None: def test_probe_payload_is_task_aware_and_reasoning_safe() -> None: - models, compiler, runtime = modules() - plan = compiler.compile_intent(generation(models), source_revision="test") + plan = compiler.compile_intent(generation(), source_revision="test") requests: list[httpx.Request] = [] def handler(request: httpx.Request) -> httpx.Response: @@ -218,6 +217,7 @@ def handler(request: httpx.Request) -> httpx.Response: with httpx.Client(transport=httpx.MockTransport(handler)) as client: receipt = runtime.probe_endpoint(plan, client=client) + assert client.is_closed is False assert receipt.passed and receipt.observed_capabilities == ("chat-completions",) payload = json.loads(requests[-1].content) assert payload["max_tokens"] == 128 @@ -225,8 +225,7 @@ def handler(request: httpx.Request) -> httpx.Response: def test_probe_uses_bearer_secret_without_serializing_it() -> None: - models, compiler, runtime = modules() - plan = compiler.compile_intent(generation(models, api_key_env="LOCAL_KEY"), source_revision="test") + plan = compiler.compile_intent(generation(api_key_env="LOCAL_KEY"), source_revision="test") def handler(request: httpx.Request) -> httpx.Response: assert request.headers["authorization"] == "Bearer test-secret" @@ -241,8 +240,7 @@ def handler(request: httpx.Request) -> httpx.Response: def test_probe_rejects_wrong_model_and_status() -> None: - models, compiler, runtime = modules() - plan = compiler.compile_intent(generation(models), source_revision="test") + plan = compiler.compile_intent(generation(), source_revision="test") def wrong_model(request: httpx.Request) -> httpx.Response: if request.url.path == "/v1/models": @@ -257,8 +255,7 @@ def wrong_model(request: httpx.Request) -> httpx.Response: def test_plan_integrity_and_pid_cleanup_are_enforced(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - models, compiler, runtime = modules() - plan = compiler.compile_intent(generation(models), source_revision="test") + plan = compiler.compile_intent(generation(), source_revision="test") with pytest.raises(compiler.PlanIntegrityError): runtime.launch_plan( plan.model_copy(update={"source_revision": "changed"}), secret_values={}, log_directory=tmp_path @@ -271,8 +268,7 @@ def test_plan_integrity_and_pid_cleanup_are_enforced(tmp_path: Path, monkeypatch def test_launch_records_process_identity_and_resolves_secrets(tmp_path: Path) -> None: - models, compiler, runtime = modules() - plan = compiler.compile_intent(generation(models, api_key_env="LOCAL_KEY"), source_revision="test") + plan = compiler.compile_intent(generation(api_key_env="LOCAL_KEY"), source_revision="test") probe = models.CapabilityProbeReceipt( plan_digest=plan.plan_digest, endpoint=plan.endpoint, @@ -295,8 +291,7 @@ def test_launch_records_process_identity_and_resolves_secrets(tmp_path: Path) -> def test_missing_secret_fails_before_process_start(tmp_path: Path) -> None: - models, compiler, runtime = modules() - plan = compiler.compile_intent(generation(models, api_key_env="LOCAL_KEY"), source_revision="test") + plan = compiler.compile_intent(generation(api_key_env="LOCAL_KEY"), source_revision="test") with mock.patch.object(runtime.subprocess, "Popen") as popen: with pytest.raises(runtime.RuntimeEffectError) as exc_info: runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) @@ -306,8 +301,7 @@ def test_missing_secret_fails_before_process_start(tmp_path: Path) -> None: def test_inspect_cancel_and_forced_cleanup_are_versioned(tmp_path: Path) -> None: - models, compiler, runtime = modules() - plan = compiler.compile_intent(generation(models), source_revision="test") + plan = compiler.compile_intent(generation(), source_revision="test") handle = models.LocalProcessHandle( external_id="4242:100", pid=4242, @@ -316,7 +310,7 @@ def test_inspect_cancel_and_forced_cleanup_are_versioned(tmp_path: Path) -> None stdout_path=str(tmp_path / "stdout.log"), stderr_path=str(tmp_path / "stderr.log"), ) - launch = launch_receipt(models, plan, handle) + launch = launch_receipt(plan, handle) with mock.patch.object(runtime, "is_handle_running", return_value=True): assert runtime.inspect_run(launch).state == "running" with ( @@ -331,7 +325,6 @@ def test_inspect_cancel_and_forced_cleanup_are_versioned(tmp_path: Path) -> None def test_process_stat_handles_spaces_and_zombies(tmp_path: Path) -> None: - models, _compiler, runtime = modules() fields = ["S", *(str(index) for index in range(4, 22)), "98765"] assert runtime._parse_process_stat(f"4242 (worker with spaces) {' '.join(fields)}") == ("S", "98765") handle = models.LocalProcessHandle( @@ -352,8 +345,7 @@ def test_process_stat_handles_spaces_and_zombies(tmp_path: Path) -> None: def test_failed_readiness_cleans_up_the_known_process(tmp_path: Path) -> None: - models, compiler, runtime = modules() - plan = compiler.compile_intent(generation(models), source_revision="test") + plan = compiler.compile_intent(generation(), source_revision="test") process = mock.Mock(pid=4242) failure = runtime.RuntimeEffectError(models.RuntimeDiagnostic(code="probe-failed", message="not ready")) with ( @@ -371,8 +363,7 @@ def test_failed_readiness_cleans_up_the_known_process(tmp_path: Path) -> None: def test_readiness_stops_when_the_managed_process_exits(tmp_path: Path) -> None: - models, compiler, runtime = modules() - plan = compiler.compile_intent(generation(models), source_revision="test") + plan = compiler.compile_intent(generation(), source_revision="test") handle = models.LocalProcessHandle( external_id="4242:100", pid=4242, diff --git a/tests/tools/test_vllm_factory.py b/tests/tools/test_vllm_factory.py index 40a66b4b..a9359928 100644 --- a/tests/tools/test_vllm_factory.py +++ b/tests/tools/test_vllm_factory.py @@ -18,11 +18,6 @@ REPO_ROOT = TOOLS_ROOT.parent -def load_factory_module(): - """Compatibility alias for the directly imported production module.""" - return factory - - def test_local_models_group_pins_vllm_and_external_factory_source() -> None: """The runtime pins vLLM and the reviewed external factory source revision.""" project = tomllib.loads((REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8")) @@ -168,7 +163,6 @@ def test_run_server_uses_the_vllm_0_27_lifecycle_boundary() -> None: api_server = importlib.import_module("vllm.entrypoints.openai.api_server") api_utils = importlib.import_module("vllm.entrypoints.serve.utils.api_utils") - factory = load_factory_module() arguments = mock.sentinel.arguments coroutine = mock.sentinel.coroutine run_vllm_server = mock.Mock(return_value=coroutine) @@ -194,8 +188,6 @@ def test_run_server_uses_the_vllm_0_27_lifecycle_boundary() -> None: def test_factory_exposes_interpreter_tools_on_path() -> None: """vLLM subprocess helpers can find executables installed beside Python.""" - factory = load_factory_module() - with ( mock.patch.object(factory.sys, "prefix", "/workspace/.venv"), mock.patch.dict(os.environ, {"PATH": "/usr/bin"}), @@ -207,7 +199,6 @@ def test_factory_exposes_interpreter_tools_on_path() -> None: def test_factory_avoids_flashinfer_jit_without_overriding_operator_choice() -> None: """The wheel-only runtime does not require a host CUDA compiler by default.""" - factory = load_factory_module() parameters = factory.VllmServerParameters(model="model", host="127.0.0.1", port=8000) with ( @@ -227,7 +218,6 @@ def test_factory_avoids_flashinfer_jit_without_overriding_operator_choice() -> N def test_factory_rejects_python_3_11_before_importing_vllm() -> None: """The server reports the local vLLM Python floor before vLLM starts.""" - factory = load_factory_module() parameters = factory.VllmServerParameters(model="model", host="127.0.0.1", port=8000) with mock.patch.object(factory.sys, "version_info", (3, 11)): @@ -237,7 +227,6 @@ def test_factory_rejects_python_3_11_before_importing_vllm() -> None: def test_factory_configures_packaged_cuda_for_flashinfer(tmp_path: Path) -> None: """The FlashInfer backend can JIT from the CUDA toolkit shipped as Python wheels.""" - factory = load_factory_module() cuda_root = tmp_path / "nvidia" / "cu13" (cuda_root / "bin").mkdir(parents=True) (cuda_root / "bin" / "nvcc").touch() @@ -262,7 +251,6 @@ def test_factory_configures_packaged_cuda_for_flashinfer(tmp_path: Path) -> None def test_factory_selects_model_and_io_plugins_together() -> None: """vLLM's shared plugin allowlist retains both factory entry-point groups.""" - factory = load_factory_module() parameters = factory.VllmServerParameters( model="nvidia/gliner-pii", revision="bd23e8ef", diff --git a/tests/tools/test_vllm_factory_adapter.py b/tests/tools/test_vllm_factory_adapter.py index fe098168..d568d099 100644 --- a/tests/tools/test_vllm_factory_adapter.py +++ b/tests/tools/test_vllm_factory_adapter.py @@ -4,17 +4,8 @@ from __future__ import annotations -from pathlib import Path - from inference_service_compiler import vllm_factory_adapter as adapter -TOOLS_ROOT = Path(__file__).resolve().parents[2] / "tools" - - -def load_adapter(): - """Compatibility alias for the directly imported production module.""" - return adapter - def test_parse_detection_request_preserves_anonymizer_options() -> None: """The adapter accepts the detector extras emitted by DataDesigner.""" @@ -53,7 +44,6 @@ def test_parse_detection_request_accepts_label_free_health_check() -> None: def test_merge_gliner_entities_restores_offsets_and_deduplicates_overlap() -> None: """Chunk-relative factory spans become stable document offsets.""" - adapter = load_adapter() chunks = [adapter.TextChunk("Alice met Bob", 0), adapter.TextChunk("Bob at NVIDIA", 10)] results = [ [ @@ -82,8 +72,6 @@ def test_merge_gliner_entities_restores_offsets_and_deduplicates_overlap() -> No def test_merge_gliner2_entities_normalizes_confidence_and_spans() -> None: """GLiNER2's schema result becomes the detector's flat entity list.""" - adapter = load_adapter() - entities = adapter.merge_entities( plugin="deberta_gliner2", chunks=[adapter.TextChunk("Email alice@example.com", 0)], @@ -117,8 +105,6 @@ def test_merge_gliner2_entities_normalizes_confidence_and_spans() -> None: def test_split_text_matches_native_character_overlap_contract() -> None: """The adapter keeps the characterized character-based chunk semantics.""" - adapter = load_adapter() - assert adapter.split_text("abcdefghij", chunk_length=6, overlap=2) == [ adapter.TextChunk("abcdef", 0), adapter.TextChunk("efghij", 4), diff --git a/tests/tools/test_vllm_factory_integration.py b/tests/tools/test_vllm_factory_integration.py index 2866d5e6..bb9d05fc 100644 --- a/tests/tools/test_vllm_factory_integration.py +++ b/tests/tools/test_vllm_factory_integration.py @@ -10,8 +10,6 @@ from inference_service_compiler import vllm_factory_integration as integration -TOOLS_ROOT = Path(__file__).resolve().parents[2] / "tools" - def test_factory_plugin_spec_is_immutable_and_complete() -> None: """Each supported plugin resolves all of its metadata through one product.""" diff --git a/tools/inference_service_compiler/compiler.py b/tools/inference_service_compiler/compiler.py index a02b4b91..6b45d145 100644 --- a/tools/inference_service_compiler/compiler.py +++ b/tools/inference_service_compiler/compiler.py @@ -76,7 +76,13 @@ class ServiceCompilation: def compile_intent(intent: InferenceIntent, *, source_revision: str) -> RunPlan: """Compile semantic intent without starting, probing, or allocating anything.""" if not source_revision: - raise ValueError("source_revision must not be empty") + raise CompilationError( + CompilerDiagnostic( + code="invalid-source-revision", + message="source_revision must not be empty", + details={}, + ) + ) required = intent.task.required_capabilities() compilation = _compile_vllm(intent) placement = intent.local @@ -136,73 +142,83 @@ def _canonical_json(value: object) -> bytes: def _compile_vllm(intent: InferenceIntent) -> ServiceCompilation: - engine = intent.vllm match intent.task: case EntityDetection() as task: - factory = engine.factory - if factory is None: - _raise_unsupported_task_engine(task.kind, "vllm") - if intent.model.revision is None: - raise CompilationError( - CompilerDiagnostic( - code="unpinned-model-revision", - message="vLLM Factory entity detection requires a pinned model revision", - details={"model": intent.model.model_id}, - ) - ) - if not supports_model(factory.plugin, intent.model.model_id): - raise CompilationError( - CompilerDiagnostic( - code="unsupported-model-engine", - message=( - f"model {intent.model.model_id!r} is not characterized for " - f"vLLM Factory plugin {factory.plugin!r}" - ), - details={"model": intent.model.model_id, "plugin": factory.plugin}, - ) - ) - if intent.model.adapter is not None: - raise CompilationError( - CompilerDiagnostic( - code="unsupported-model-adapter", - message="vLLM Factory entity detection does not support a model adapter", - details={"engine": "vllm", "task": task.kind}, - ) - ) - command, runtime = _vllm_command(intent, engine) - return ServiceCompilation( - command=command, - runtime=runtime, - declared_capabilities=task.required_capabilities(), - compatibility_evidence=( - CompatibilityEvidence( - rule="vllm-factory-entity-detection-v1", - outcome="runtime-probe-required", - detail=( - "vLLM Factory supplies model preparation, pooling inference, and IO processing; " - "the Anonymizer adapter preserves dynamic labels, offsets, and scores" - ), - ), + return _compile_entity_detection(intent, task) + case Generation() as task: + return _compile_generation(intent, task) + case _: + assert_never(intent.task) + + +def _compile_entity_detection(intent: InferenceIntent, task: EntityDetection) -> ServiceCompilation: + _validate_factory_detection(intent, task) + command, runtime = _vllm_command(intent, intent.vllm) + return ServiceCompilation( + command=command, + runtime=runtime, + declared_capabilities=task.required_capabilities(), + compatibility_evidence=( + CompatibilityEvidence( + rule="vllm-factory-entity-detection-v1", + outcome="runtime-probe-required", + detail=( + "vLLM Factory supplies model preparation, pooling inference, and IO processing; " + "the Anonymizer adapter preserves dynamic labels, offsets, and scores" ), + ), + ), + ) + + +def _validate_factory_detection(intent: InferenceIntent, task: EntityDetection) -> None: + factory = intent.vllm.factory + if factory is None: + _raise_unsupported_task_engine(task.kind, "vllm") + if intent.model.revision is None: + raise CompilationError( + CompilerDiagnostic( + code="unpinned-model-revision", + message="vLLM Factory entity detection requires a pinned model revision", + details={"model": intent.model.model_id}, ) - case Generation() as task: - if engine.factory is not None: - _raise_unsupported_task_engine(task.kind, "vllm") - command, runtime = _vllm_command(intent, engine) - return ServiceCompilation( - command=command, - runtime=runtime, - declared_capabilities=("chat-completions",), - compatibility_evidence=( - CompatibilityEvidence( - rule="vllm-openai-compatible-v1", - outcome="characterized", - detail="vLLM exposes chat completions for generation", - ), + ) + if not supports_model(factory.plugin, intent.model.model_id): + raise CompilationError( + CompilerDiagnostic( + code="unsupported-model-engine", + message=( + f"model {intent.model.model_id!r} is not characterized for vLLM Factory plugin {factory.plugin!r}" ), + details={"model": intent.model.model_id, "plugin": factory.plugin}, ) - case _: - assert_never(intent.task) + ) + if intent.model.adapter is not None: + raise CompilationError( + CompilerDiagnostic( + code="unsupported-model-adapter", + message="vLLM Factory entity detection does not support a model adapter", + details={"engine": "vllm", "task": task.kind}, + ) + ) + + +def _compile_generation(intent: InferenceIntent, task: Generation) -> ServiceCompilation: + if intent.vllm.factory is not None: + _raise_unsupported_task_engine(task.kind, "vllm") + command, runtime = _vllm_command(intent, intent.vllm) + return ServiceCompilation( + command=command, + runtime=runtime, + declared_capabilities=("chat-completions",), + compatibility_evidence=( + CompatibilityEvidence( + rule="vllm-openai-compatible-v1", + outcome="characterized", + detail="vLLM exposes chat completions for generation", + ), + ), + ) def _vllm_command( @@ -234,22 +250,16 @@ def _vllm_engine_arguments(intent: InferenceIntent, engine: Vllm) -> tuple[Comma intent.model.revision, ) ) - if engine.served_model_name is not None: - arguments.extend(_literal_arguments("--served-model-name", engine.served_model_name)) - if engine.tensor_parallel_size is not None: - arguments.extend(_literal_arguments("--tensor-parallel-size", str(engine.tensor_parallel_size))) - if engine.gpu_memory_utilization is not None: - arguments.extend(_literal_arguments("--gpu-memory-utilization", str(engine.gpu_memory_utilization))) - if engine.max_model_len is not None: - arguments.extend(_literal_arguments("--max-model-len", str(engine.max_model_len))) - if engine.max_num_seqs is not None: - arguments.extend(_literal_arguments("--max-num-seqs", str(engine.max_num_seqs))) - if engine.eager: - arguments.extend(_literal_arguments("--enforce-eager")) - if engine.enable_prefix_caching: - arguments.extend(_literal_arguments("--enable-prefix-caching")) - if engine.async_scheduling: - arguments.extend(_literal_arguments("--async-scheduling")) + arguments.extend(_optional_vllm_arguments(engine)) + arguments.extend( + LiteralArgument(value=flag) + for flag, enabled in ( + ("--enforce-eager", engine.eager), + ("--enable-prefix-caching", engine.enable_prefix_caching), + ("--async-scheduling", engine.async_scheduling), + ) + if enabled + ) if engine.mamba_backend is not None: arguments.extend(_literal_arguments("--mamba-backend", engine.mamba_backend)) if engine.mamba_ssm_cache_dtype != "auto": @@ -278,6 +288,19 @@ def _vllm_engine_arguments(intent: InferenceIntent, engine: Vllm) -> tuple[Comma return tuple(arguments) +def _optional_vllm_arguments(engine: Vllm) -> tuple[CommandArgument, ...]: + values = ( + ("--served-model-name", engine.served_model_name), + ("--tensor-parallel-size", engine.tensor_parallel_size), + ("--gpu-memory-utilization", engine.gpu_memory_utilization), + ("--max-model-len", engine.max_model_len), + ("--max-num-seqs", engine.max_num_seqs), + ) + return tuple( + argument for flag, value in values if value is not None for argument in _literal_arguments(flag, str(value)) + ) + + def _vllm_environment(engine: Vllm) -> tuple[SecretEnvironmentVariable, ...]: if engine.api_key_env is None: return () diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index 1a50fd24..675b1b29 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -10,6 +10,7 @@ import subprocess import time from collections.abc import Mapping +from contextlib import nullcontext from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path @@ -57,24 +58,20 @@ def probe_endpoint( """Probe one managed endpoint and record only capabilities observed at runtime.""" verify_plan(plan) headers = _probe_headers(plan, secret_values or {}) - owns_client = client is None - active_client = client or httpx.Client(timeout=10) try: - models_response = active_client.get(plan.readiness.url, headers=headers) - if models_response.status_code != plan.readiness.expected_status: - raise ValueError( - f"readiness probe returned status {models_response.status_code}, " - f"expected {plan.readiness.expected_status}" - ) - models = _parse_models(models_response.json()) - observed = _probe_task(plan, active_client, headers) + with nullcontext(client) if client is not None else httpx.Client(timeout=10) as active_client: + models_response = active_client.get(plan.readiness.url, headers=headers) + if models_response.status_code != plan.readiness.expected_status: + raise ValueError( + f"readiness probe returned status {models_response.status_code}, " + f"expected {plan.readiness.expected_status}" + ) + models = _parse_models(models_response.json()) + observed = _probe_task(plan, active_client, headers) except (httpx.HTTPError, KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: raise RuntimeEffectError( RuntimeDiagnostic(code="probe-failed", message=f"capability probe failed: {exc}") ) from exc - finally: - if owns_client: - active_client.close() return CapabilityProbeReceipt( plan_digest=plan.plan_digest, endpoint=plan.endpoint, From 52afb253358f424623be15edff578be69909f909 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Fri, 14 Aug 2026 16:06:12 +0000 Subject: [PATCH 15/28] refactor(dev): clarify local inference lifecycle semantics Signed-off-by: Aaron Gonzales --- README.md | 4 +- docs/concepts/inference-services.md | 33 ++-- docs/concepts/self-hosting-gliner.md | 4 +- .../posts/self-hosted-anonymizer-b300.md | 7 +- skills/anonymizer/SKILL.md | 2 +- tests/tools/test_inference_service.py | 98 ++++++---- tools/inference_service.py | 2 +- tools/inference_service_compiler/cli.py | 22 +-- tools/inference_service_compiler/compiler.py | 185 +++++++----------- tools/inference_service_compiler/models.py | 70 +++---- tools/inference_service_compiler/profiles.py | 6 +- tools/inference_service_compiler/runtime.py | 67 +++++-- tools/inference_service_profiles/gliner2.toml | 2 +- .../gpt-oss-120b.toml | 3 +- .../gpt-oss-20b.toml | 3 +- .../nemotron-3.5-lightning.toml | 3 +- .../nvidia-gliner.toml | 2 +- .../qwen3-30b-a3b-instruct.toml | 3 +- .../vllm-local.toml | 3 +- 19 files changed, 260 insertions(+), 259 deletions(-) diff --git a/README.md b/README.md index bd76d6ab..540bdfe0 100644 --- a/README.md +++ b/README.md @@ -163,8 +163,8 @@ Factory project, or for vLLM generation: ```bash uv run tools/inference_service.py compile --profile tools/inference_service_profiles/nvidia-gliner.toml --source-revision 3f68c145 --output plan.json uv run tools/inference_service.py launch --plan plan.json --output launch.json -uv run tools/inference_service.py inspect --receipt launch.json -uv run tools/inference_service.py cancel --receipt launch.json +uv run tools/inference_service.py status --receipt launch.json +uv run tools/inference_service.py stop --receipt launch.json ``` The tool is not part of the wheel and does not attach to externally owned diff --git a/docs/concepts/inference-services.md b/docs/concepts/inference-services.md index 6ac4966e..8c8ac32b 100644 --- a/docs/concepts/inference-services.md +++ b/docs/concepts/inference-services.md @@ -6,8 +6,8 @@ NeMo Anonymizer can run its detector and generation roles against models on your own NVIDIA GPU. The repository includes pinned profiles and a lifecycle tool for this purpose. The tool compiles a profile into an immutable plan, -starts the service, proves its API contract, and records enough process identity -to inspect or stop it later. +starts the service, checks readiness, and records the local process handle it +observed so it can later report status or request cleanup. This path suits development workstations, on-premises servers, and isolated environments where text must stay on local infrastructure. It manages one @@ -48,12 +48,12 @@ The workflow has two durable files: ```text profile.toml -> compile -> plan.json -> launch -> launch.json | | - +-> probe +-> inspect - +-> cancel + +-> probe +-> status + +-> stop ``` `compile` has no runtime effects. The plan contains the full command, endpoint, -dependencies, compatibility evidence, and a SHA-256 digest. Runtime commands +dependencies, compatibility assessments, and a SHA-256 digest. Runtime commands verify that digest before acting. `launch` waits for `/v1/models` and a task-specific request. A generation @@ -101,13 +101,15 @@ uv run --python 3.12 python tools/inference_service.py launch \ --log-directory .inference-service-runs ``` -The command returns after both probes pass. Keep `gliner-launch.json`; it owns -the process identity used for inspection and cleanup. +The command returns after both readiness checks pass. Keep `gliner-launch.json`: +it is an unsigned durable operation record containing the observed handle and +its consistency fingerprint. It does not own or prove process identity; later +status and stop operations re-check the exact PID and start marker. ### Operate the service ```bash -uv run --python 3.12 python tools/inference_service.py inspect \ +uv run --python 3.12 python tools/inference_service.py status \ --receipt gliner-launch.json uv run --python 3.12 python tools/inference_service.py probe \ @@ -115,12 +117,13 @@ uv run --python 3.12 python tools/inference_service.py probe \ curl -sf http://127.0.0.1:8001/v1/models | python -m json.tool -uv run --python 3.12 python tools/inference_service.py cancel \ +uv run --python 3.12 python tools/inference_service.py stop \ --receipt gliner-launch.json ``` -`cancel` sends `SIGTERM` to the receipt-owned process group, waits for the -profile's shutdown timeout, and uses `SIGKILL` if the group remains alive. +`stop` checks the recorded PID and start marker before sending `SIGTERM` to its +process group, waits for the profile's shutdown timeout, and uses `SIGKILL` if +the group remains alive. ## Deploy in a GPU container @@ -202,7 +205,7 @@ on the host. Operate the service through `docker exec`: ```bash docker exec anonymizer-local-models \ - python tools/inference_service.py inspect \ + python tools/inference_service.py status \ --receipt /state/gliner-launch.json docker exec anonymizer-local-models \ @@ -214,11 +217,11 @@ curl -sf http://127.0.0.1:8001/v1/models | python -m json.tool ### Stop cleanly -Cancel each managed service before removing the container: +Stop each managed service before removing the container: ```bash docker exec anonymizer-local-models \ - python tools/inference_service.py cancel \ + python tools/inference_service.py stop \ --receipt /state/gliner-launch.json docker stop anonymizer-local-models @@ -226,7 +229,7 @@ docker rm anonymizer-local-models ``` Stopping the container first removes the runtime boundary before the tool can -write a cancellation receipt. Use `cancel` first when lifecycle evidence or +write a stop receipt. Use `stop` first when lifecycle evidence or graceful model shutdown matters. ## Connect Anonymizer diff --git a/docs/concepts/self-hosting-gliner.md b/docs/concepts/self-hosting-gliner.md index f780c75e..aa073330 100644 --- a/docs/concepts/self-hosting-gliner.md +++ b/docs/concepts/self-hosting-gliner.md @@ -125,8 +125,8 @@ complete host and GPU container workflows. NVIDIA GLiNER uses `deberta_gliner`; GLiNER2 uses `deberta_gliner2`. Launch writes a versioned receipt only after the model-list and detection -contract probes pass. Use that receipt with the compiler's `inspect` and -`cancel` commands instead of supervising the internal server module directly. +contract probes pass. Use that receipt with the compiler's `status` and +`stop` commands instead of supervising the internal server module directly. The model families do not use identical label vocabularies. The request example below targets the default NVIDIA model and uses `user_name`; the default GLiNER2 PII checkpoint uses `username` for that category. diff --git a/docs/devnotes/posts/self-hosted-anonymizer-b300.md b/docs/devnotes/posts/self-hosted-anonymizer-b300.md index c3d5de3c..ced7fad1 100644 --- a/docs/devnotes/posts/self-hosted-anonymizer-b300.md +++ b/docs/devnotes/posts/self-hosted-anonymizer-b300.md @@ -135,11 +135,12 @@ The `CUDA_ROOT` path above is specific to the Brev B300 SXM6 environment used fo GLiNER ran on the same machine. Current reruns use the source-tree local-model deployment tool described in [Self-hosting GLiNER](../../concepts/self-hosting-gliner.md). -It records the exact model, vLLM settings, endpoint, and process identity in -versioned plans and receipts. The current detector path uses vLLM Factory. +It records the exact model, vLLM settings, endpoint, and observed process +coordinates in versioned plans and receipts. The current detector path uses +vLLM Factory. ```toml title="gliner-b300.toml" -schema_version = "inference-service.intent/v2" +schema_version = "inference-service.local-spec/v2" [task] kind = "entity-detection" diff --git a/skills/anonymizer/SKILL.md b/skills/anonymizer/SKILL.md index ad43017a..b040b1ec 100644 --- a/skills/anonymizer/SKILL.md +++ b/skills/anonymizer/SKILL.md @@ -73,7 +73,7 @@ read `docs/troubleshooting.md` or the - **`anonymizer` not installed:** Tell the user `nemo-anonymizer` is not in this Python environment (requires Python ≥ 3.11). Ask if they want you to install it (`pip install nemo-anonymizer`) or do it themselves. Do not install without permission. - **Model/provider setup:** Plain `Anonymizer()` ships with bundled `models.yaml` and `providers.yaml` (see `src/anonymizer/config/default_model_configs/`). For the default path, confirm `NVIDIA_API_KEY` is set. Pass custom `model_configs` or `model_providers` only for non-default endpoints or model pools. See `docs/concepts/models.md` or the [published models guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/models/). - **LLM calls failing at preview:** Check for a missing or invalid API key, a network problem, or a wrong endpoint URL. See `docs/troubleshooting.md` "Validation passed but `preview` errors at LLM call" or the [published troubleshooting guide](https://nvidia-nemo.github.io/Anonymizer/dev/troubleshooting/). -- **Local / on-prem GLiNER:** Clone the Anonymizer repository, then deploy `tools/inference_service_profiles/nvidia-gliner.toml` on the GPU host or through `tools/inference_service.Dockerfile`. Add a provider with the plan's endpoint (normally `http://localhost:8001/v1`) and point `gliner-pii-detector` at `provider: local-gliner`. The vLLM Factory adapter supports DataDesigner's health check, so do not suppress it. Preflight errors about missing aliases usually mean `model_configs` lists only the detector. Include the full default pool. Use the launch receipt to inspect or cancel the managed process before stopping its container. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published deployment guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). +- **Local / on-prem GLiNER:** Clone the Anonymizer repository, then deploy `tools/inference_service_profiles/nvidia-gliner.toml` on the GPU host or through `tools/inference_service.Dockerfile`. Add a provider with the plan's endpoint (normally `http://localhost:8001/v1`) and point `gliner-pii-detector` at `provider: local-gliner`. The vLLM Factory adapter supports DataDesigner's health check, so do not suppress it. Preflight errors about missing aliases usually mean `model_configs` lists only the detector. Include the full default pool. Use the launch receipt with `status` to observe the managed process or `stop` to clean it up before stopping its container. See [`docs/concepts/inference-services.md`](../../docs/concepts/inference-services.md) or the [published deployment guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/inference-services/), plus [`docs/concepts/self-hosting-gliner.md`](../../docs/concepts/self-hosting-gliner.md) or the [published self-hosting guide](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/). # Output Template diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index 2870226a..7a7f565b 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -22,8 +22,8 @@ CLI = TOOLS / "inference_service.py" -def generation(**vllm: object) -> models.InferenceIntent: - return models.InferenceIntent( +def generation(**vllm: object) -> models.LocalInferenceServiceSpec: + return models.LocalInferenceServiceSpec( task=models.Generation(), model=models.HuggingFaceModel(model_id="openai/gpt-oss-20b", revision="abc"), vllm=models.Vllm.model_validate(vllm), @@ -44,7 +44,7 @@ def launch_receipt( plan_digest=plan.plan_digest, endpoint=plan.endpoint, observed_at="2026-08-07T00:00:00+00:00", - models=(plan.expected_model,), + models=(plan.served_model_name,), observed_capabilities=plan.required_capabilities, passed=True, ), @@ -58,10 +58,25 @@ def test_cli_remains_a_directly_executable_source_entrypoint() -> None: def test_all_shipped_profiles_compile() -> None: plans = [ - compiler.compile_intent(load_profile(path), source_revision="test") for path in sorted(PROFILES.glob("*.toml")) + compiler.compile_profile(load_profile(path), source_revision="test") for path in sorted(PROFILES.glob("*.toml")) ] - assert len(plans) == 7 + assert plans assert all(plan.schema_version == "inference-service.run-plan/v2" for plan in plans) + assert all(plan.readiness.path == "/models" for plan in plans) + assert all(plan.served_model_name for plan in plans) + + +def test_plan_keeps_one_endpoint_address_and_one_served_model_vocabulary() -> None: + plan = compiler.compile_profile(generation(served_model_name="local-generator"), source_revision="test") + + serialized = plan.model_dump(mode="json") + assert plan.endpoint.url == "http://127.0.0.1:8000/v1" + assert plan.readiness.path == "/models" + assert plan.served_model_name == "local-generator" + assert "host" not in serialized["readiness"] + assert "port" not in serialized["readiness"] + assert "intent_digest" not in serialized + assert "declared_capabilities" not in serialized def test_compile_command_writes_a_digest_verified_plan(tmp_path: Path) -> None: @@ -80,7 +95,7 @@ def test_compile_command_writes_a_digest_verified_plan(tmp_path: Path) -> None: ) assert exc_info.value.code == 0 plan = compiler.load_plan(output.read_text(encoding="utf-8")) - assert plan.expected_model == "anonymizer-local" + assert plan.served_model_name == "anonymizer-local" def test_compile_command_translates_non_directory_profile_paths(tmp_path: Path) -> None: @@ -119,7 +134,7 @@ def test_compile_command_translates_empty_source_revision() -> None: def test_generation_argv_keeps_local_vllm_controls_and_omits_defaults() -> None: - plan = compiler.compile_intent( + plan = compiler.compile_profile( generation(api_key_env="LOCAL_KEY", tensor_parallel_size=2, max_model_len=4096, eager=True), source_revision="test", ) @@ -135,15 +150,15 @@ def test_generation_argv_keeps_local_vllm_controls_and_omits_defaults() -> None: def test_factory_detection_is_task_bounded() -> None: - valid = models.InferenceIntent( + valid = models.LocalInferenceServiceSpec( task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), model=models.HuggingFaceModel(model_id="nvidia/gliner-pii", revision="abc"), vllm=models.Vllm(factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), local=models.LocalProcess(), ) - assert "--vllm-factory-plugin" in compiler.compile_intent(valid, source_revision="test").command.render_argv() + assert "--vllm-factory-plugin" in compiler.compile_profile(valid, source_revision="test").command.render_argv() with pytest.raises(compiler.CompilationError, match="does not support"): - compiler.compile_intent( + compiler.compile_profile( generation(factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), source_revision="test" ) @@ -159,21 +174,21 @@ def test_factory_detection_requires_a_pin_and_characterized_model() -> None: ("nvidia/gliner-pii", None, "pinned model revision"), ("unknown/model", "abc", "not characterized"), ): - intent = models.InferenceIntent( + spec = models.LocalInferenceServiceSpec( task=models.EntityDetection(dynamic_labels=True, offsets=True, scores=True), model=models.HuggingFaceModel(model_id=model_id, revision=revision), vllm=models.Vllm(factory=models.VllmFactoryIntegration(plugin="deberta_gliner")), local=models.LocalProcess(), ) with pytest.raises(compiler.CompilationError, match=message): - compiler.compile_intent(intent, source_revision="test") + compiler.compile_profile(spec, source_revision="test") def test_removed_domains_are_invalid_profile_fields() -> None: with pytest.raises(ValidationError): - models.InferenceIntent.model_validate( + models.LocalInferenceServiceSpec.model_validate( { - "schema_version": "inference-service.intent/v2", + "schema_version": "inference-service.local-spec/v2", "task": {"kind": "generation"}, "model": {"model_id": "x"}, "vllm": {}, @@ -184,7 +199,7 @@ def test_removed_domains_are_invalid_profile_fields() -> None: def test_plan_digest_detects_transport_mutation() -> None: - plan = compiler.compile_intent(generation(), source_revision="test") + plan = compiler.compile_profile(generation(), source_revision="test") changed = json.loads(plan.model_dump_json()) changed["endpoint"]["port"] = 9000 with pytest.raises(compiler.PlanIntegrityError, match="plan digest mismatch"): @@ -192,7 +207,7 @@ def test_plan_digest_detects_transport_mutation() -> None: def test_lora_is_rendered_as_a_model_artifact() -> None: - intent = models.InferenceIntent( + spec = models.LocalInferenceServiceSpec( task=models.Generation(), model=models.HuggingFaceModel( model_id="openai/gpt-oss-20b", @@ -201,12 +216,12 @@ def test_lora_is_rendered_as_a_model_artifact() -> None: vllm=models.Vllm(), local=models.LocalProcess(), ) - argv = compiler.compile_intent(intent, source_revision="test").command.render_argv() + argv = compiler.compile_profile(spec, source_revision="test").command.render_argv() assert argv[-3:] == ("--enable-lora", "--lora-modules", "privacy=/models/privacy-adapter") def test_probe_payload_is_task_aware_and_reasoning_safe() -> None: - plan = compiler.compile_intent(generation(), source_revision="test") + plan = compiler.compile_profile(generation(), source_revision="test") requests: list[httpx.Request] = [] def handler(request: httpx.Request) -> httpx.Response: @@ -225,12 +240,12 @@ def handler(request: httpx.Request) -> httpx.Response: def test_probe_uses_bearer_secret_without_serializing_it() -> None: - plan = compiler.compile_intent(generation(api_key_env="LOCAL_KEY"), source_revision="test") + plan = compiler.compile_profile(generation(api_key_env="LOCAL_KEY"), source_revision="test") def handler(request: httpx.Request) -> httpx.Response: assert request.headers["authorization"] == "Bearer test-secret" if request.url.path == "/v1/models": - return httpx.Response(200, json={"data": [{"id": plan.expected_model}]}) + return httpx.Response(200, json={"data": [{"id": plan.served_model_name}]}) return httpx.Response(200, json={"choices": [{"message": {"content": "ready"}}]}) with httpx.Client(transport=httpx.MockTransport(handler)) as client: @@ -240,7 +255,7 @@ def handler(request: httpx.Request) -> httpx.Response: def test_probe_rejects_wrong_model_and_status() -> None: - plan = compiler.compile_intent(generation(), source_revision="test") + plan = compiler.compile_profile(generation(), source_revision="test") def wrong_model(request: httpx.Request) -> httpx.Response: if request.url.path == "/v1/models": @@ -255,7 +270,7 @@ def wrong_model(request: httpx.Request) -> httpx.Response: def test_plan_integrity_and_pid_cleanup_are_enforced(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - plan = compiler.compile_intent(generation(), source_revision="test") + plan = compiler.compile_profile(generation(), source_revision="test") with pytest.raises(compiler.PlanIntegrityError): runtime.launch_plan( plan.model_copy(update={"source_revision": "changed"}), secret_values={}, log_directory=tmp_path @@ -268,12 +283,12 @@ def test_plan_integrity_and_pid_cleanup_are_enforced(tmp_path: Path, monkeypatch def test_launch_records_process_identity_and_resolves_secrets(tmp_path: Path) -> None: - plan = compiler.compile_intent(generation(api_key_env="LOCAL_KEY"), source_revision="test") + plan = compiler.compile_profile(generation(api_key_env="LOCAL_KEY"), source_revision="test") probe = models.CapabilityProbeReceipt( plan_digest=plan.plan_digest, endpoint=plan.endpoint, observed_at="2026-08-07T00:00:00+00:00", - models=(plan.expected_model,), + models=(plan.served_model_name,), observed_capabilities=("chat-completions",), passed=True, ) @@ -290,8 +305,25 @@ def test_launch_records_process_identity_and_resolves_secrets(tmp_path: Path) -> assert receipt.handle.external_id == "4242:100" +def test_launch_refuses_an_unmarked_process_and_cleans_up(tmp_path: Path) -> None: + plan = compiler.compile_profile(generation(), source_revision="test") + process = mock.Mock(pid=4242) + process.poll.return_value = 0 + with ( + mock.patch.object(runtime.subprocess, "Popen", return_value=process), + mock.patch.object(runtime, "read_process_start_marker", return_value=None), + mock.patch.object(runtime.os, "killpg") as killpg, + pytest.raises(runtime.RuntimeEffectError) as exc_info, + ): + runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) + killpg.assert_called_once_with(4242, runtime.signal.SIGTERM) + process.wait.assert_called_once_with(timeout=1) + assert exc_info.value.diagnostic.code == "missing-process-start-marker" + assert exc_info.value.diagnostic.cleanup_complete is True + + def test_missing_secret_fails_before_process_start(tmp_path: Path) -> None: - plan = compiler.compile_intent(generation(api_key_env="LOCAL_KEY"), source_revision="test") + plan = compiler.compile_profile(generation(api_key_env="LOCAL_KEY"), source_revision="test") with mock.patch.object(runtime.subprocess, "Popen") as popen: with pytest.raises(runtime.RuntimeEffectError) as exc_info: runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) @@ -300,8 +332,8 @@ def test_missing_secret_fails_before_process_start(tmp_path: Path) -> None: assert exc_info.value.diagnostic.known_effects == () -def test_inspect_cancel_and_forced_cleanup_are_versioned(tmp_path: Path) -> None: - plan = compiler.compile_intent(generation(), source_revision="test") +def test_status_stop_and_forced_cleanup_are_versioned(tmp_path: Path) -> None: + plan = compiler.compile_profile(generation(), source_revision="test") handle = models.LocalProcessHandle( external_id="4242:100", pid=4242, @@ -312,15 +344,15 @@ def test_inspect_cancel_and_forced_cleanup_are_versioned(tmp_path: Path) -> None ) launch = launch_receipt(plan, handle) with mock.patch.object(runtime, "is_handle_running", return_value=True): - assert runtime.inspect_run(launch).state == "running" + assert runtime.status_run(launch).state == "running" with ( mock.patch.object(runtime, "is_handle_running", side_effect=[True, True, False]), mock.patch.object(runtime.time, "monotonic", side_effect=[0.0, 0.0, 1.0]), mock.patch.object(runtime.os, "killpg") as killpg, ): - canceled = runtime.cancel_run(launch) - assert canceled.outcome == "forced" - assert canceled.cleanup_complete is True + stopped = runtime.stop_run(launch) + assert stopped.outcome == "forced" + assert stopped.cleanup_complete is True assert [call.args[1] for call in killpg.call_args_list] == [runtime.signal.SIGTERM, runtime.signal.SIGKILL] @@ -345,7 +377,7 @@ def test_process_stat_handles_spaces_and_zombies(tmp_path: Path) -> None: def test_failed_readiness_cleans_up_the_known_process(tmp_path: Path) -> None: - plan = compiler.compile_intent(generation(), source_revision="test") + plan = compiler.compile_profile(generation(), source_revision="test") process = mock.Mock(pid=4242) failure = runtime.RuntimeEffectError(models.RuntimeDiagnostic(code="probe-failed", message="not ready")) with ( @@ -363,7 +395,7 @@ def test_failed_readiness_cleans_up_the_known_process(tmp_path: Path) -> None: def test_readiness_stops_when_the_managed_process_exits(tmp_path: Path) -> None: - plan = compiler.compile_intent(generation(), source_revision="test") + plan = compiler.compile_profile(generation(), source_revision="test") handle = models.LocalProcessHandle( external_id="4242:100", pid=4242, diff --git a/tools/inference_service.py b/tools/inference_service.py index 99468eee..0e6958af 100755 --- a/tools/inference_service.py +++ b/tools/inference_service.py @@ -10,7 +10,7 @@ # "structlog>=24.4", # ] # /// -"""Compile and manage local inference services from typed intent.""" +"""Compile and manage local inference services from typed specifications.""" from __future__ import annotations diff --git a/tools/inference_service_compiler/cli.py b/tools/inference_service_compiler/cli.py index 6ef70666..1c61f7be 100644 --- a/tools/inference_service_compiler/cli.py +++ b/tools/inference_service_compiler/cli.py @@ -15,15 +15,15 @@ import cyclopts from pydantic import BaseModel, ValidationError -from inference_service_compiler.compiler import CompilationError, PlanIntegrityError, compile_intent, load_plan +from inference_service_compiler.compiler import CompilationError, PlanIntegrityError, compile_profile, load_plan from inference_service_compiler.models import LaunchReceipt from inference_service_compiler.profiles import load_profile from inference_service_compiler.runtime import ( RuntimeEffectError, - cancel_run, - inspect_run, launch_plan, probe_endpoint, + status_run, + stop_run, ) app = cyclopts.App(help="Compile and manage local inference services from TOML profiles.") @@ -67,7 +67,7 @@ def compile_plan( ) -> None: """Compile a v2 TOML profile without performing runtime effects.""" parsed = load_profile(profile) - plan = compile_intent(parsed, source_revision=source_revision) + plan = compile_profile(parsed, source_revision=source_revision) write_json(plan, output) @@ -98,20 +98,20 @@ def probe(*, plan: Path, output: Path | None = None) -> None: write_json(probe_endpoint(parsed, secret_values=secret_values), output) -@app.command(name="inspect") +@app.command(name="status") @command_errors -def inspect_command(*, receipt: Path, output: Path | None = None) -> None: - """Inspect the reconnectable identity in a launch receipt.""" +def status_command(*, receipt: Path, output: Path | None = None) -> None: + """Record the current state observed for a launch receipt handle.""" launch_receipt = LaunchReceipt.model_validate_json(receipt.read_text(encoding="utf-8")) - write_json(inspect_run(launch_receipt), output) + write_json(status_run(launch_receipt), output) @app.command @command_errors -def cancel(*, receipt: Path, output: Path | None = None) -> None: - """Cancel and clean up the managed identity in a launch receipt.""" +def stop(*, receipt: Path, output: Path | None = None) -> None: + """Stop and clean up the process group recorded by a launch receipt.""" launch_receipt = LaunchReceipt.model_validate_json(receipt.read_text(encoding="utf-8")) - write_json(cancel_run(launch_receipt), output) + write_json(stop_run(launch_receipt), output) def write_json(value: BaseModel, output: Path | None) -> None: diff --git a/tools/inference_service_compiler/compiler.py b/tools/inference_service_compiler/compiler.py index 6b45d145..7b8ae73d 100644 --- a/tools/inference_service_compiler/compiler.py +++ b/tools/inference_service_compiler/compiler.py @@ -1,30 +1,24 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Pure compilation from inference intent to an immutable run plan.""" +"""Pure compilation from inference spec to an immutable run plan.""" from __future__ import annotations import hashlib import hmac import json -from dataclasses import dataclass from typing import Never, assert_never -from pydantic import BaseModel - from inference_service_compiler.models import ( - Capability, - CommandArgument, CommandSpec, - CompatibilityEvidence, - EndpointContract, + CompatibilityAssessment, + EndpointAddress, EntityDetection, FrozenModel, Generation, - HttpProbe, - InferenceIntent, LiteralArgument, - LocalProcessRuntime, + LocalInferenceServiceSpec, + ReadinessCheck, RunPlan, SecretEnvironmentVariable, Vllm, @@ -44,7 +38,7 @@ class CompilerDiagnostic(FrozenModel): - """Serializable reason that semantic intent cannot be compiled.""" + """Serializable reason that semantic spec cannot be compiled.""" code: str message: str @@ -63,18 +57,8 @@ class PlanIntegrityError(ValueError): """A serialized plan does not match its declared digest.""" -@dataclass(frozen=True, slots=True) -class ServiceCompilation: - """Complete immutable product of selecting one service implementation.""" - - command: CommandSpec - runtime: LocalProcessRuntime - declared_capabilities: tuple[Capability, ...] - compatibility_evidence: tuple[CompatibilityEvidence, ...] - - -def compile_intent(intent: InferenceIntent, *, source_revision: str) -> RunPlan: - """Compile semantic intent without starting, probing, or allocating anything.""" +def compile_profile(spec: LocalInferenceServiceSpec, *, source_revision: str) -> RunPlan: + """Compile semantic spec without starting, probing, or allocating anything.""" if not source_revision: raise CompilationError( CompilerDiagnostic( @@ -83,40 +67,29 @@ def compile_intent(intent: InferenceIntent, *, source_revision: str) -> RunPlan: details={}, ) ) - required = intent.task.required_capabilities() - compilation = _compile_vllm(intent) - placement = intent.local - endpoint = EndpointContract(host=placement.host, port=placement.port) + required = spec.task.required_capabilities() + command, assessments = _compile_vllm(spec) + placement = spec.local + endpoint = EndpointAddress(host=placement.host, port=placement.port) plan = RunPlan( plan_digest="", - intent_digest=digest_model(intent), - intent=intent, - command=compilation.command, - runtime=compilation.runtime, + spec=spec, + command=command, endpoint=endpoint, - readiness=HttpProbe( - host=placement.host, - port=placement.port, - path="/v1/models", - timeout_seconds=intent.local.startup_timeout_seconds, - bearer_token_environment_variable=intent.vllm.api_key_env, + readiness=ReadinessCheck( + path="/models", + timeout_seconds=spec.local.startup_timeout_seconds, + bearer_token_environment_variable=spec.vllm.api_key_env, ), - expected_model=intent.expected_model, + served_model_name=spec.served_model_name, required_capabilities=required, - declared_capabilities=compilation.declared_capabilities, - compatibility_evidence=compilation.compatibility_evidence, - dependencies=_plan_dependencies(intent), + compatibility_assessments=assessments, + dependencies=_plan_dependencies(spec), source_revision=source_revision, ) return plan.model_copy(update={"plan_digest": digest_plan(plan)}) -def digest_model(model: BaseModel) -> str: - """Return a stable SHA-256 digest of one typed transport value.""" - payload = model.model_dump(mode="json") - return hashlib.sha256(_canonical_json(payload)).hexdigest() - - def digest_plan(plan: RunPlan) -> str: """Return the stable digest of a plan excluding its digest field.""" payload = plan.model_dump(mode="json", exclude={"plan_digest"}) @@ -141,59 +114,56 @@ def _canonical_json(value: object) -> bytes: return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode() -def _compile_vllm(intent: InferenceIntent) -> ServiceCompilation: - match intent.task: +def _compile_vllm(spec: LocalInferenceServiceSpec) -> tuple[CommandSpec, tuple[CompatibilityAssessment, ...]]: + match spec.task: case EntityDetection() as task: - return _compile_entity_detection(intent, task) + return _compile_entity_detection(spec, task) case Generation() as task: - return _compile_generation(intent, task) + return _compile_generation(spec, task) case _: - assert_never(intent.task) - - -def _compile_entity_detection(intent: InferenceIntent, task: EntityDetection) -> ServiceCompilation: - _validate_factory_detection(intent, task) - command, runtime = _vllm_command(intent, intent.vllm) - return ServiceCompilation( - command=command, - runtime=runtime, - declared_capabilities=task.required_capabilities(), - compatibility_evidence=( - CompatibilityEvidence( - rule="vllm-factory-entity-detection-v1", - outcome="runtime-probe-required", - detail=( - "vLLM Factory supplies model preparation, pooling inference, and IO processing; " - "the Anonymizer adapter preserves dynamic labels, offsets, and scores" - ), + assert_never(spec.task) + + +def _compile_entity_detection( + spec: LocalInferenceServiceSpec, task: EntityDetection +) -> tuple[CommandSpec, tuple[CompatibilityAssessment, ...]]: + _validate_factory_detection(spec, task) + command = _vllm_command(spec, spec.vllm) + return command, ( + CompatibilityAssessment( + rule="vllm-factory-entity-detection-v1", + outcome="runtime-probe-required", + detail=( + "vLLM Factory supplies model preparation, pooling inference, and IO processing; " + "the Anonymizer adapter preserves dynamic labels, offsets, and scores" ), ), ) -def _validate_factory_detection(intent: InferenceIntent, task: EntityDetection) -> None: - factory = intent.vllm.factory +def _validate_factory_detection(spec: LocalInferenceServiceSpec, task: EntityDetection) -> None: + factory = spec.vllm.factory if factory is None: _raise_unsupported_task_engine(task.kind, "vllm") - if intent.model.revision is None: + if spec.model.revision is None: raise CompilationError( CompilerDiagnostic( code="unpinned-model-revision", message="vLLM Factory entity detection requires a pinned model revision", - details={"model": intent.model.model_id}, + details={"model": spec.model.model_id}, ) ) - if not supports_model(factory.plugin, intent.model.model_id): + if not supports_model(factory.plugin, spec.model.model_id): raise CompilationError( CompilerDiagnostic( code="unsupported-model-engine", message=( - f"model {intent.model.model_id!r} is not characterized for vLLM Factory plugin {factory.plugin!r}" + f"model {spec.model.model_id!r} is not characterized for vLLM Factory plugin {factory.plugin!r}" ), - details={"model": intent.model.model_id, "plugin": factory.plugin}, + details={"model": spec.model.model_id, "plugin": factory.plugin}, ) ) - if intent.model.adapter is not None: + if spec.model.adapter is not None: raise CompilationError( CompilerDiagnostic( code="unsupported-model-adapter", @@ -203,51 +173,48 @@ def _validate_factory_detection(intent: InferenceIntent, task: EntityDetection) ) -def _compile_generation(intent: InferenceIntent, task: Generation) -> ServiceCompilation: - if intent.vllm.factory is not None: +def _compile_generation( + spec: LocalInferenceServiceSpec, task: Generation +) -> tuple[CommandSpec, tuple[CompatibilityAssessment, ...]]: + if spec.vllm.factory is not None: _raise_unsupported_task_engine(task.kind, "vllm") - command, runtime = _vllm_command(intent, intent.vllm) - return ServiceCompilation( - command=command, - runtime=runtime, - declared_capabilities=("chat-completions",), - compatibility_evidence=( - CompatibilityEvidence( - rule="vllm-openai-compatible-v1", - outcome="characterized", - detail="vLLM exposes chat completions for generation", - ), + command = _vllm_command(spec, spec.vllm) + return command, ( + CompatibilityAssessment( + rule="vllm-openai-compatible-v1", + outcome="characterized", + detail="vLLM exposes chat completions for generation", ), ) def _vllm_command( - intent: InferenceIntent, + spec: LocalInferenceServiceSpec, engine: Vllm, -) -> tuple[CommandSpec, LocalProcessRuntime]: - engine_arguments = _vllm_engine_arguments(intent, engine) - placement = intent.local +) -> CommandSpec: + engine_arguments = _vllm_engine_arguments(spec, engine) + placement = spec.local argv = _literal_arguments( engine.python_executable, "tools/inference_service_compiler/vllm_server.py", - intent.model.model_id, + spec.model.model_id, "--host", placement.host, "--port", str(placement.port), ) - return CommandSpec(argv=argv + engine_arguments, environment=_vllm_environment(engine)), LocalProcessRuntime() + return CommandSpec(argv=argv + engine_arguments, environment=_vllm_environment(engine)) -def _vllm_engine_arguments(intent: InferenceIntent, engine: Vllm) -> tuple[CommandArgument, ...]: - arguments: list[CommandArgument] = [] - if intent.model.revision is not None: +def _vllm_engine_arguments(spec: LocalInferenceServiceSpec, engine: Vllm) -> tuple[LiteralArgument, ...]: + arguments: list[LiteralArgument] = [] + if spec.model.revision is not None: arguments.extend( _literal_arguments( "--revision", - intent.model.revision, + spec.model.revision, "--tokenizer-revision", - intent.model.revision, + spec.model.revision, ) ) arguments.extend(_optional_vllm_arguments(engine)) @@ -277,18 +244,18 @@ def _vllm_engine_arguments(intent: InferenceIntent, engine: Vllm) -> tuple[Comma engine.factory.prepared_model_root, ) ) - if intent.model.adapter is not None: + if spec.model.adapter is not None: arguments.extend( _literal_arguments( "--enable-lora", "--lora-modules", - f"{intent.model.adapter.name}={intent.model.adapter.path}", + f"{spec.model.adapter.name}={spec.model.adapter.path}", ) ) return tuple(arguments) -def _optional_vllm_arguments(engine: Vllm) -> tuple[CommandArgument, ...]: +def _optional_vllm_arguments(engine: Vllm) -> tuple[LiteralArgument, ...]: values = ( ("--served-model-name", engine.served_model_name), ("--tensor-parallel-size", engine.tensor_parallel_size), @@ -312,15 +279,15 @@ def _vllm_environment(engine: Vllm) -> tuple[SecretEnvironmentVariable, ...]: ) -def _literal_arguments(*values: str) -> tuple[CommandArgument, ...]: +def _literal_arguments(*values: str) -> tuple[LiteralArgument, ...]: return tuple(LiteralArgument(value=value) for value in values) -def _plan_dependencies(intent: InferenceIntent) -> tuple[str, ...]: +def _plan_dependencies(spec: LocalInferenceServiceSpec) -> tuple[str, ...]: dependencies = [VLLM_DEPENDENCY] - if intent.vllm.factory is not None: + if spec.vllm.factory is not None: dependencies.append(VLLM_FACTORY_DEPENDENCY) - if intent.vllm.mamba_backend == "flashinfer": + if spec.vllm.mamba_backend == "flashinfer": dependencies.extend(FLASHINFER_CUDA_TOOLCHAIN_DEPENDENCIES) return tuple(dependencies) diff --git a/tools/inference_service_compiler/models.py b/tools/inference_service_compiler/models.py index 39dcb119..84a6ecf6 100644 --- a/tools/inference_service_compiler/models.py +++ b/tools/inference_service_compiler/models.py @@ -9,12 +9,12 @@ from pydantic import BaseModel, ConfigDict, Field -INTENT_SCHEMA_VERSION = "inference-service.intent/v2" +SPEC_SCHEMA_VERSION = "inference-service.local-spec/v2" PLAN_SCHEMA_VERSION = "inference-service.run-plan/v2" CAPABILITY_PROBE_RECEIPT_SCHEMA_VERSION = "inference-service.capability-probe-receipt/v1" LAUNCH_RECEIPT_SCHEMA_VERSION = "inference-service.launch-receipt/v1" STATUS_RECEIPT_SCHEMA_VERSION = "inference-service.status-receipt/v1" -CANCELLATION_RECEIPT_SCHEMA_VERSION = "inference-service.cancellation-receipt/v1" +STOP_RECEIPT_SCHEMA_VERSION = "inference-service.stop-receipt/v1" Capability = Literal["chat-completions", "dynamic-labels", "offsets", "scores"] FactoryPlugin = Literal["deberta_gliner", "deberta_gliner2"] @@ -59,7 +59,6 @@ class Generation(FrozenModel): """Text-generation requirements independent of a serving engine.""" kind: Literal["generation"] = "generation" - chat: Literal[True] = True def required_capabilities(self) -> tuple[Capability, ...]: """Return the endpoint capabilities required by this task.""" @@ -121,17 +120,17 @@ class LocalProcess(FrozenModel): shutdown_timeout_seconds: float = Field(default=30, gt=0) -class InferenceIntent(FrozenModel): +class LocalInferenceServiceSpec(FrozenModel): """Complete semantic input to pure inference-service compilation.""" - schema_version: Literal["inference-service.intent/v2"] = INTENT_SCHEMA_VERSION + schema_version: Literal["inference-service.local-spec/v2"] = SPEC_SCHEMA_VERSION task: TaskSpec model: HuggingFaceModel vllm: Vllm local: LocalProcess @property - def expected_model(self) -> str: + def served_model_name(self) -> str: """Return the model ID that the compiled endpoint must serve.""" if self.model.adapter is not None: return self.model.adapter.name @@ -147,9 +146,6 @@ class LiteralArgument(FrozenModel): value: str -CommandArgument = LiteralArgument - - class EnvironmentVariable(FrozenModel): """One ordinary non-secret process environment value.""" @@ -172,7 +168,7 @@ class SecretEnvironmentVariable(FrozenModel): class CommandSpec(FrozenModel): """Complete argv and ordinary environment for one managed service.""" - argv: tuple[CommandArgument, ...] = Field(min_length=1) + argv: tuple[LiteralArgument, ...] = Field(min_length=1) environment: tuple[EnvironmentSpec, ...] = () working_directory: str = "." @@ -190,7 +186,7 @@ def secret_sources(self) -> tuple[str, ...]: ) def render_environment(self) -> dict[str, str]: - """Render an inspectable environment with every secret source redacted.""" + """Render an observable environment with every secret source redacted.""" values: dict[str, str] = {} for variable in self.environment: match variable: @@ -219,7 +215,7 @@ def resolve_environment(self, secret_values: Mapping[str, str]) -> dict[str, str return resolved -class EndpointContract(FrozenModel): +class EndpointAddress(FrozenModel): """Direct OpenAI-compatible endpoint produced by a run.""" scheme: Literal["http"] = "http" @@ -233,31 +229,16 @@ def url(self) -> str: return f"{self.scheme}://{self.host}:{self.port}{self.base_path}" -class HttpProbe(FrozenModel): +class ReadinessCheck(FrozenModel): """One bounded readiness or capability-probe request.""" - scheme: Literal["http"] = "http" - host: str - port: int path: str expected_status: int = 200 timeout_seconds: float = Field(gt=0) bearer_token_environment_variable: str | None = Field(default=None, min_length=1) - @property - def url(self) -> str: - """Return the complete probe URL.""" - return f"{self.scheme}://{self.host}:{self.port}{self.path}" - - -class LocalProcessRuntime(FrozenModel): - """Runtime facts needed to launch and stop a local process.""" - - kind: Literal["local-process"] = "local-process" - cleanup: Literal["terminate-process-group"] = "terminate-process-group" - -class CompatibilityEvidence(FrozenModel): +class CompatibilityAssessment(FrozenModel): """One compiler rule supporting or qualifying the selected combination.""" rule: str @@ -270,16 +251,13 @@ class RunPlan(FrozenModel): schema_version: Literal["inference-service.run-plan/v2"] = PLAN_SCHEMA_VERSION plan_digest: str - intent_digest: str = Field(min_length=1) - intent: InferenceIntent + spec: LocalInferenceServiceSpec command: CommandSpec - runtime: LocalProcessRuntime - endpoint: EndpointContract - readiness: HttpProbe - expected_model: str = Field(min_length=1) + endpoint: EndpointAddress + readiness: ReadinessCheck + served_model_name: str = Field(min_length=1) required_capabilities: tuple[Capability, ...] - declared_capabilities: tuple[Capability, ...] - compatibility_evidence: tuple[CompatibilityEvidence, ...] + compatibility_assessments: tuple[CompatibilityAssessment, ...] dependencies: tuple[str, ...] = () source_revision: str = Field(min_length=1) @@ -289,7 +267,7 @@ class CapabilityProbeReceipt(FrozenModel): schema_version: Literal["inference-service.capability-probe-receipt/v1"] = CAPABILITY_PROBE_RECEIPT_SCHEMA_VERSION plan_digest: str - endpoint: EndpointContract + endpoint: EndpointAddress observed_at: str models: tuple[str, ...] observed_capabilities: tuple[Capability, ...] @@ -297,19 +275,19 @@ class CapabilityProbeReceipt(FrozenModel): class LocalProcessHandle(FrozenModel): - """Reconnectable identity for a managed local process.""" + """Observed local-process coordinates checked against PID reuse.""" kind: Literal["local-process"] = "local-process" external_id: str pid: int = Field(ge=1) process_group_id: int = Field(ge=1) - start_marker: str | None + start_marker: str = Field(min_length=1) stdout_path: str stderr_path: str class LaunchReceipt(FrozenModel): - """Known launch effects, reconnectable identity, and readiness evidence.""" + """Durable record of a launch operation and its readiness observations.""" schema_version: Literal["inference-service.launch-receipt/v1"] = LAUNCH_RECEIPT_SCHEMA_VERSION plan_digest: str @@ -320,7 +298,7 @@ class LaunchReceipt(FrozenModel): class StatusReceipt(FrozenModel): - """Observed state for a reconnectable managed handle.""" + """Recorded state observation for a managed handle.""" schema_version: Literal["inference-service.status-receipt/v1"] = STATUS_RECEIPT_SCHEMA_VERSION plan_digest: str @@ -329,12 +307,12 @@ class StatusReceipt(FrozenModel): state: Literal["running", "stopped"] -class CancellationReceipt(FrozenModel): - """Cancellation outcome and cleanup state for a managed handle.""" +class StopReceipt(FrozenModel): + """Recorded cleanup outcome for a managed handle.""" - schema_version: Literal["inference-service.cancellation-receipt/v1"] = CANCELLATION_RECEIPT_SCHEMA_VERSION + schema_version: Literal["inference-service.stop-receipt/v1"] = STOP_RECEIPT_SCHEMA_VERSION plan_digest: str - canceled_at: str + stopped_at: str handle: LocalProcessHandle outcome: Literal["terminated", "already-stopped", "forced"] cleanup_complete: bool diff --git a/tools/inference_service_compiler/profiles.py b/tools/inference_service_compiler/profiles.py index f00bd4d9..b3ca7d56 100644 --- a/tools/inference_service_compiler/profiles.py +++ b/tools/inference_service_compiler/profiles.py @@ -7,10 +7,10 @@ import tomllib from pathlib import Path -from inference_service_compiler.models import InferenceIntent +from inference_service_compiler.models import LocalInferenceServiceSpec -def load_profile(path: Path) -> InferenceIntent: +def load_profile(path: Path) -> LocalInferenceServiceSpec: """Load and validate one TOML inference-service profile.""" payload = tomllib.loads(path.read_text(encoding="utf-8")) - return InferenceIntent.model_validate(payload) + return LocalInferenceServiceSpec.model_validate(payload) diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index 675b1b29..a76ab69d 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -20,7 +20,6 @@ from inference_service_compiler.compiler import verify_plan from inference_service_compiler.models import ( - CancellationReceipt, Capability, CapabilityProbeReceipt, EntityDetection, @@ -30,6 +29,7 @@ RunPlan, RuntimeDiagnostic, StatusReceipt, + StopReceipt, ) @@ -60,7 +60,7 @@ def probe_endpoint( headers = _probe_headers(plan, secret_values or {}) try: with nullcontext(client) if client is not None else httpx.Client(timeout=10) as active_client: - models_response = active_client.get(plan.readiness.url, headers=headers) + models_response = active_client.get(f"{plan.endpoint.url}{plan.readiness.path}", headers=headers) if models_response.status_code != plan.readiness.expected_status: raise ValueError( f"readiness probe returned status {models_response.status_code}, " @@ -78,7 +78,7 @@ def probe_endpoint( observed_at=_now(), models=models, observed_capabilities=observed, - passed=plan.expected_model in models and set(plan.required_capabilities).issubset(observed), + passed=plan.served_model_name in models and set(plan.required_capabilities).issubset(observed), ) @@ -103,7 +103,7 @@ def launch_plan( return LaunchReceipt( plan_digest=plan.plan_digest, launched_at=_now(), - shutdown_timeout_seconds=plan.intent.local.shutdown_timeout_seconds, + shutdown_timeout_seconds=plan.spec.local.shutdown_timeout_seconds, handle=handle, probe=probe, ) @@ -117,7 +117,7 @@ def _probe_or_cleanup( try: probe = wait_for_readiness(plan, secret_values=secret_values, handle=handle) except RuntimeEffectError as exc: - cleanup_complete = _cleanup_handle(handle, plan.intent.local.shutdown_timeout_seconds) + cleanup_complete = _cleanup_handle(handle, plan.spec.local.shutdown_timeout_seconds) raise RuntimeEffectError( exc.diagnostic.model_copy( update={ @@ -127,7 +127,7 @@ def _probe_or_cleanup( ) ) from exc if not probe.passed: - cleanup_complete = _cleanup_handle(handle, plan.intent.local.shutdown_timeout_seconds) + cleanup_complete = _cleanup_handle(handle, plan.spec.local.shutdown_timeout_seconds) raise RuntimeEffectError( RuntimeDiagnostic( code="capability-mismatch", @@ -139,8 +139,8 @@ def _probe_or_cleanup( return probe -def inspect_run(launch: LaunchReceipt) -> StatusReceipt: - """Inspect a reconnectable handle without changing its state.""" +def status_run(launch: LaunchReceipt) -> StatusReceipt: + """Observe a recorded handle without changing process state.""" state = "running" if is_handle_running(launch.handle) else "stopped" return StatusReceipt( plan_digest=launch.plan_digest, @@ -150,21 +150,21 @@ def inspect_run(launch: LaunchReceipt) -> StatusReceipt: ) -def cancel_run(launch: LaunchReceipt) -> CancellationReceipt: +def stop_run(launch: LaunchReceipt) -> StopReceipt: """Stop the exact process group recorded by a launch receipt.""" handle = launch.handle if not is_handle_running(handle): - return CancellationReceipt( + return StopReceipt( plan_digest=launch.plan_digest, - canceled_at=_now(), + stopped_at=_now(), handle=handle, outcome="already-stopped", cleanup_complete=True, ) stop = _stop_running_handle(handle, launch.shutdown_timeout_seconds) - return CancellationReceipt( + return StopReceipt( plan_digest=launch.plan_digest, - canceled_at=_now(), + stopped_at=_now(), handle=handle, outcome=stop.outcome, cleanup_complete=stop.cleanup_complete, @@ -174,7 +174,7 @@ def cancel_run(launch: LaunchReceipt) -> CancellationReceipt: def is_handle_running(handle: LocalProcessHandle) -> bool: """Check the external identity while guarding against Linux PID reuse.""" current_marker = read_process_start_marker(handle.pid) - if handle.start_marker is not None and current_marker != handle.start_marker: + if current_marker != handle.start_marker: return False if read_process_state(handle.pid) == "Z": return False @@ -230,7 +230,7 @@ def wait_for_readiness( last_error: RuntimeEffectError | None = None while time.monotonic() < deadline: if handle is not None and not is_handle_running(handle): - log_hint = f"; inspect {handle.stderr_path}" + log_hint = f"; status {handle.stderr_path}" raise RuntimeEffectError( RuntimeDiagnostic( code="launch-exited", @@ -279,12 +279,12 @@ def _parse_process_stat(payload: str) -> tuple[str, str] | None: def _probe_task(plan: RunPlan, client: httpx.Client, headers: Mapping[str, str]) -> tuple[Capability, ...]: - match plan.intent.task: + match plan.spec.task: case Generation(): response = client.post( f"{plan.endpoint.url}/chat/completions", json={ - "model": plan.expected_model, + "model": plan.served_model_name, "messages": [{"role": "user", "content": "Reply with the word ready."}], "max_tokens": 128, "chat_template_kwargs": { @@ -303,7 +303,7 @@ def _probe_task(plan: RunPlan, client: httpx.Client, headers: Mapping[str, str]) response = client.post( f"{plan.endpoint.url}/chat/completions", json={ - "model": plan.expected_model, + "model": plan.served_model_name, "messages": [{"role": "user", "content": "Ada Lovelace"}], "labels": ["person"], "threshold": 0.1, @@ -323,7 +323,7 @@ def _probe_task(plan: RunPlan, client: httpx.Client, headers: Mapping[str, str]) observed.append("scores") return tuple(observed) case _: - assert_never(plan.intent.task) + assert_never(plan.spec.task) def _parse_models(payload: object) -> tuple[str, ...]: @@ -373,9 +373,18 @@ def _launch_process( start_new_session=True, ) marker = read_process_start_marker(process.pid) - suffix = marker or "unknown" + if marker is None: + _terminate_unmarked_launch(process) + raise RuntimeEffectError( + RuntimeDiagnostic( + code="missing-process-start-marker", + message="cannot record a managed process without a Linux start marker", + known_effects=(str(process.pid),), + cleanup_complete=process.poll() is not None, + ) + ) return LocalProcessHandle( - external_id=f"{process.pid}:{suffix}", + external_id=f"{process.pid}:{marker}", pid=process.pid, process_group_id=process.pid, start_marker=marker, @@ -384,5 +393,21 @@ def _launch_process( ) +def _terminate_unmarked_launch(process: subprocess.Popen[bytes]) -> None: + """Clean up a just-created child when its durable identity cannot be recorded.""" + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + try: + process.wait(timeout=1) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + return + process.wait() + + def _now() -> str: return datetime.now(UTC).isoformat() diff --git a/tools/inference_service_profiles/gliner2.toml b/tools/inference_service_profiles/gliner2.toml index 391db093..30b6868e 100644 --- a/tools/inference_service_profiles/gliner2.toml +++ b/tools/inference_service_profiles/gliner2.toml @@ -1,4 +1,4 @@ -schema_version = "inference-service.intent/v2" +schema_version = "inference-service.local-spec/v2" [task] kind = "entity-detection" diff --git a/tools/inference_service_profiles/gpt-oss-120b.toml b/tools/inference_service_profiles/gpt-oss-120b.toml index 7c4af546..3b6d7676 100644 --- a/tools/inference_service_profiles/gpt-oss-120b.toml +++ b/tools/inference_service_profiles/gpt-oss-120b.toml @@ -1,8 +1,7 @@ -schema_version = "inference-service.intent/v2" +schema_version = "inference-service.local-spec/v2" [task] kind = "generation" -chat = true [model] model_id = "openai/gpt-oss-120b" diff --git a/tools/inference_service_profiles/gpt-oss-20b.toml b/tools/inference_service_profiles/gpt-oss-20b.toml index d9d65d92..e33dfdbc 100644 --- a/tools/inference_service_profiles/gpt-oss-20b.toml +++ b/tools/inference_service_profiles/gpt-oss-20b.toml @@ -1,8 +1,7 @@ -schema_version = "inference-service.intent/v2" +schema_version = "inference-service.local-spec/v2" [task] kind = "generation" -chat = true [model] model_id = "openai/gpt-oss-20b" diff --git a/tools/inference_service_profiles/nemotron-3.5-lightning.toml b/tools/inference_service_profiles/nemotron-3.5-lightning.toml index cf924252..6131711d 100644 --- a/tools/inference_service_profiles/nemotron-3.5-lightning.toml +++ b/tools/inference_service_profiles/nemotron-3.5-lightning.toml @@ -1,8 +1,7 @@ -schema_version = "inference-service.intent/v2" +schema_version = "inference-service.local-spec/v2" [task] kind = "generation" -chat = true [model] model_id = "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16" diff --git a/tools/inference_service_profiles/nvidia-gliner.toml b/tools/inference_service_profiles/nvidia-gliner.toml index f62357e3..f7024aac 100644 --- a/tools/inference_service_profiles/nvidia-gliner.toml +++ b/tools/inference_service_profiles/nvidia-gliner.toml @@ -1,4 +1,4 @@ -schema_version = "inference-service.intent/v2" +schema_version = "inference-service.local-spec/v2" [task] kind = "entity-detection" diff --git a/tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml b/tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml index a1b50f91..399c60d4 100644 --- a/tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml +++ b/tools/inference_service_profiles/qwen3-30b-a3b-instruct.toml @@ -1,8 +1,7 @@ -schema_version = "inference-service.intent/v2" +schema_version = "inference-service.local-spec/v2" [task] kind = "generation" -chat = true [model] model_id = "Qwen/Qwen3-30B-A3B-Instruct-2507" diff --git a/tools/inference_service_profiles/vllm-local.toml b/tools/inference_service_profiles/vllm-local.toml index 3f9aec39..bfbb5650 100644 --- a/tools/inference_service_profiles/vllm-local.toml +++ b/tools/inference_service_profiles/vllm-local.toml @@ -1,8 +1,7 @@ -schema_version = "inference-service.intent/v2" +schema_version = "inference-service.local-spec/v2" [task] kind = "generation" -chat = true [model] model_id = "TinyLlama/TinyLlama-1.1B-Chat-v1.0" From 5e0af17856f17506c3ab5990333dc8f7e3ca53c2 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Fri, 14 Aug 2026 16:33:55 +0000 Subject: [PATCH 16/28] fix(dev): bound local inference cleanup Signed-off-by: Aaron Gonzales --- README.md | 2 +- docs/concepts/inference-services.md | 17 ++++++++++----- tests/tools/test_inference_service.py | 23 +++++++++++++++++++- tools/inference_service_compiler/cli.py | 2 +- tools/inference_service_compiler/compiler.py | 8 +++---- tools/inference_service_compiler/models.py | 2 +- tools/inference_service_compiler/runtime.py | 22 +++++++++---------- 7 files changed, 50 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 540bdfe0..103f5470 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ make install-pre-commit # Install pre-commit hooks ### Local inference services -Use the source-tree local-model deployment tool to create immutable plans and +Use the source-tree local-model deployment tool to create checksummed plans and managed launch receipts for GLiNER or GLiNER2 through the pinned external vLLM Factory project, or for vLLM generation: diff --git a/docs/concepts/inference-services.md b/docs/concepts/inference-services.md index 8c8ac32b..07c00303 100644 --- a/docs/concepts/inference-services.md +++ b/docs/concepts/inference-services.md @@ -5,9 +5,9 @@ NeMo Anonymizer can run its detector and generation roles against models on your own NVIDIA GPU. The repository includes pinned profiles and a lifecycle -tool for this purpose. The tool compiles a profile into an immutable plan, -starts the service, checks readiness, and records the local process handle it -observed so it can later report status or request cleanup. +tool for this purpose. The tool compiles a profile into a frozen, checksummed +plan, starts the service, checks readiness, and records the local process handle +it observed so it can later report status or request cleanup. This path suits development workstations, on-premises servers, and isolated environments where text must stay on local infrastructure. It manages one @@ -56,6 +56,11 @@ profile.toml -> compile -> plan.json -> launch -> launch.json dependencies, compatibility assessments, and a SHA-256 digest. Runtime commands verify that digest before acting. +The digest detects accidental changes while a plan is stored or transported. +This developer tool trusts whoever can write the profile and plan: it does not +authenticate the plan's author, sign the plan, or recompile the embedded spec to +prove that every derived field is semantically consistent. + `launch` waits for `/v1/models` and a task-specific request. A generation service must return a chat completion. A detector must demonstrate dynamic labels, offsets, and scores. The launch receipt records the process group and @@ -264,6 +269,6 @@ every alias needed by the selected pipeline. See For the detector request and response contract, chunking behavior, and a live PII request, see [Self-hosting GLiNER](self-hosting-gliner.md). -Compilation proves static compatibility. The launch probe proves the endpoint -shape. Run Anonymizer preview and evaluation before accepting a model for a -privacy-sensitive workload. +Compilation records static compatibility assessments. The launch probe observes +the endpoint shape and required task behavior. Run Anonymizer preview and +evaluation before accepting a model for a privacy-sensitive workload. diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index 7a7f565b..a9ae4339 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -163,7 +163,7 @@ def test_factory_detection_is_task_bounded() -> None: ) -def test_factory_plugin_is_closed_at_the_intent_boundary() -> None: +def test_factory_plugin_is_closed_at_the_spec_boundary() -> None: """Profiles cannot select an uncharacterized Factory plugin.""" with pytest.raises(ValidationError): models.VllmFactoryIntegration.model_validate({"plugin": "unsupported"}) @@ -322,6 +322,27 @@ def test_launch_refuses_an_unmarked_process_and_cleans_up(tmp_path: Path) -> Non assert exc_info.value.diagnostic.cleanup_complete is True +def test_launch_refuses_an_unmarked_process_when_cleanup_cannot_reap(tmp_path: Path) -> None: + plan = compiler.compile_profile(generation(), source_revision="test") + process = mock.Mock(pid=4242) + process.poll.return_value = None + process.wait.side_effect = ( + runtime.subprocess.TimeoutExpired(cmd=plan.command.render_argv(), timeout=1), + runtime.subprocess.TimeoutExpired(cmd=plan.command.render_argv(), timeout=1), + ) + with ( + mock.patch.object(runtime.subprocess, "Popen", return_value=process), + mock.patch.object(runtime, "read_process_start_marker", return_value=None), + mock.patch.object(runtime.os, "killpg") as killpg, + pytest.raises(runtime.RuntimeEffectError) as exc_info, + ): + runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) + assert killpg.call_args_list == [mock.call(4242, runtime.signal.SIGTERM), mock.call(4242, runtime.signal.SIGKILL)] + assert all(call.kwargs["timeout"] > 0 for call in process.wait.call_args_list) + assert exc_info.value.diagnostic.code == "missing-process-start-marker" + assert exc_info.value.diagnostic.cleanup_complete is False + + def test_missing_secret_fails_before_process_start(tmp_path: Path) -> None: plan = compiler.compile_profile(generation(api_key_env="LOCAL_KEY"), source_revision="test") with mock.patch.object(runtime.subprocess, "Popen") as popen: diff --git a/tools/inference_service_compiler/cli.py b/tools/inference_service_compiler/cli.py index 1c61f7be..5305804c 100644 --- a/tools/inference_service_compiler/cli.py +++ b/tools/inference_service_compiler/cli.py @@ -79,7 +79,7 @@ def launch( output: Path | None = None, log_directory: Path = Path(".inference-service-runs"), ) -> None: - """Launch a compiled plan and write its reconnectable handle receipt.""" + """Launch a compiled plan and record its observed handle and readiness.""" parsed = load_plan(plan.read_text(encoding="utf-8")) secret_values = {name: os.environ[name] for name in parsed.command.secret_sources if name in os.environ} write_json( diff --git a/tools/inference_service_compiler/compiler.py b/tools/inference_service_compiler/compiler.py index 7b8ae73d..a594ecca 100644 --- a/tools/inference_service_compiler/compiler.py +++ b/tools/inference_service_compiler/compiler.py @@ -46,7 +46,7 @@ class CompilerDiagnostic(FrozenModel): class CompilationError(ValueError): - """Intent failed a closed compiler compatibility rule.""" + """A service spec failed a closed compiler compatibility rule.""" def __init__(self, diagnostic: CompilerDiagnostic) -> None: super().__init__(diagnostic.message) @@ -54,7 +54,7 @@ def __init__(self, diagnostic: CompilerDiagnostic) -> None: class PlanIntegrityError(ValueError): - """A serialized plan does not match its declared digest.""" + """A serialized plan does not match its declared consistency checksum.""" def compile_profile(spec: LocalInferenceServiceSpec, *, source_revision: str) -> RunPlan: @@ -97,14 +97,14 @@ def digest_plan(plan: RunPlan) -> str: def load_plan(serialized: str | bytes) -> RunPlan: - """Parse a closed v2 plan and reject transport mutation.""" + """Parse a closed v2 plan and reject accidental transport mutation.""" plan = RunPlan.model_validate_json(serialized) verify_plan(plan) return plan def verify_plan(plan: RunPlan) -> None: - """Reject a plan whose declared digest does not match its contents.""" + """Check transport consistency, without authenticating or recompiling a plan.""" expected = digest_plan(plan) if not hmac.compare_digest(plan.plan_digest, expected): raise PlanIntegrityError(f"plan digest mismatch: declared {plan.plan_digest!r}, computed {expected!r}") diff --git a/tools/inference_service_compiler/models.py b/tools/inference_service_compiler/models.py index 84a6ecf6..e0806166 100644 --- a/tools/inference_service_compiler/models.py +++ b/tools/inference_service_compiler/models.py @@ -247,7 +247,7 @@ class CompatibilityAssessment(FrozenModel): class RunPlan(FrozenModel): - """Portable, immutable, effect-free instructions for one service run.""" + """Portable, frozen, effect-free instructions for one service run.""" schema_version: Literal["inference-service.run-plan/v2"] = PLAN_SCHEMA_VERSION plan_digest: str diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index a76ab69d..88754b70 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Explicit local-process effects for immutable run plans.""" +"""Explicit local-process effects for frozen, checksummed run plans.""" from __future__ import annotations @@ -88,7 +88,7 @@ def launch_plan( secret_values: dict[str, str], log_directory: Path, ) -> LaunchReceipt: - """Launch a verified plan and return reconnectable identity plus readiness evidence.""" + """Launch a checksummed plan and record its observed handle and readiness.""" verify_plan(plan) try: command_environment = plan.command.resolve_environment(secret_values) @@ -394,19 +394,17 @@ def _launch_process( def _terminate_unmarked_launch(process: subprocess.Popen[bytes]) -> None: - """Clean up a just-created child when its durable identity cannot be recorded.""" - try: - os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - return - try: - process.wait(timeout=1) - except subprocess.TimeoutExpired: + """Make a bounded cleanup attempt when a child cannot be recorded.""" + for signal_to_send in (signal.SIGTERM, signal.SIGKILL): try: - os.killpg(process.pid, signal.SIGKILL) + os.killpg(process.pid, signal_to_send) except ProcessLookupError: return - process.wait() + try: + process.wait(timeout=1) + except subprocess.TimeoutExpired: + continue + return def _now() -> str: From b58eec76ad5028425a41045fb5dc8bc453573724 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Fri, 14 Aug 2026 18:51:22 +0000 Subject: [PATCH 17/28] docs(dev): document W&B scorecard workflow Signed-off-by: Aaron Gonzales --- tools/measurement/README.md | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/tools/measurement/README.md b/tools/measurement/README.md index 08922da5..71e5a528 100644 --- a/tools/measurement/README.md +++ b/tools/measurement/README.md @@ -475,6 +475,68 @@ Strict online imports require `--wandb-entity` (or `ANONYMIZER_MEASUREMENT_WANDB_ENTITY`) so the result always identifies the destination and includes a W&B run URL. Offline imports may omit the entity. +### NVIDIA PR vs. Main Scorecard + +The NVIDIA benchmark project has a saved +[PR vs. main scorecard](https://wandb.ai/nemo-llm-service/nemo-anonymizer-benchmarks/reports/PR-vs-main-scorecard--VmlldzoxNzY4NTMzMA). +The report does not query every benchmark run directly. It displays one active +dashboard snapshot generated from eligible imported runs. + +A benchmark run enters the scorecard snapshot only when all of these conditions +hold: + +- The W&B run state is `finished`. +- `benchmark_role` is `main-baseline` or `candidate`. +- `anonymizer_mode` is `replace` or `rewrite`. +- The run config contains `anonymizer_config_id`, and `benchmark_config_ids` + resolves to at least one config ID. +- The run summary contains `publication/complete=true`. + +The scorecard matches main and candidate runs by `anonymizer_config_id` and +`benchmark_config_ids`. It separates datasets by `benchmark_workload_ids`. +Use the same values for a comparison pair; PR candidates also require a PR +number. The sealed case supplies `benchmark_config_ids` and +`benchmark_workload_ids`. The importer command supplies the remaining identity +fields. + +Append fields like these to the strict importer command above for a main +baseline: + +```bash + --benchmark-role main-baseline \ + --benchmark-kind main \ + --branch main \ + --commit-sha 0123456789abcdef0123456789abcdef01234567 \ + --anonymizer-config-id rat-replace-throughput \ + --anonymizer-mode replace +``` + +Use matching comparison identifiers for the candidate: + +```bash + --benchmark-role candidate \ + --benchmark-kind pr \ + --branch contributor/feat/local-models \ + --commit-sha 89abcdef0123456789abcdef0123456789abcdef \ + --pr-number 212 \ + --anonymizer-config-id rat-replace-throughput \ + --anonymizer-mode replace +``` + +The strict sealed importer is the only repository command that currently emits +the completion marker and accepts the full scorecard identity. Native runs from +`run_benchmarks.py --wandb-mode online` do not set those fields and will not +appear in this scorecard. They remain available in the W&B runs table and in +workspaces created by `create_wandb_report.py`. + +After publishing both sides, rerun the scorecard snapshot publisher. That +publisher is not shipped in this repository. The active W&B dashboard run +retains its source as `code/publish_pr_scorecard.py`, but the publisher also +depends on a team-owned HTML template and saved-view state. Until those assets +move into the repository, coordinate the refresh with the dashboard publisher +owner. A successful benchmark import does not refresh the saved scorecard by +itself. + The goal of W&B support is to get sanitized benchmark data into W&B. Workspaces, reports, views, and panels are presentation layers on top of that data. They can be edited in W&B, regenerated with the tooling below, or replaced as the From 8a52581c2f71910b91ddc9e3fa3964f041076b3d Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 20 Aug 2026 17:27:57 +0000 Subject: [PATCH 18/28] fix: bound vLLM pooling request fan-out Signed-off-by: Aaron Gonzales --- docs/concepts/self-hosting-gliner.md | 12 +- tests/tools/test_vllm_factory_adapter.py | 104 +++++++++++++++++- .../vllm_factory_adapter.py | 62 ++++++++--- 3 files changed, 161 insertions(+), 17 deletions(-) diff --git a/docs/concepts/self-hosting-gliner.md b/docs/concepts/self-hosting-gliner.md index aa073330..b5532936 100644 --- a/docs/concepts/self-hosting-gliner.md +++ b/docs/concepts/self-hosting-gliner.md @@ -74,12 +74,20 @@ inference itself. It handles two wire-level responsibilities: entities with document offsets and overlap deduplication. ```python title="tools/inference_service_compiler/vllm_factory_adapter.py (excerpt)" -results = await asyncio.gather( - *(invoke_pooling(handler=handler, text=chunk, ...) for chunk, offset in chunks) +results = await invoke_pooling_chunks( + handler=handler, + detection=detection, + chunks=chunks, + ... ) entities = merge_entities(plugin=plugin, chunks=chunks, results=results, ...) ``` +The adapter rejects a labeled request that would produce more than 256 chunks +before it materializes chunk text or calls vLLM. Accepted requests run at most +eight pooling calls concurrently; the remaining admitted chunks wait for a +worker. + When `flat_ner` is `false` (Anonymizer's default), the adapter removes nested subset spans before score-based deduplication across chunk overlaps. A request without `labels` returns an empty entity list so DataDesigner's generic model diff --git a/tests/tools/test_vllm_factory_adapter.py b/tests/tools/test_vllm_factory_adapter.py index d568d099..7ea3b627 100644 --- a/tests/tools/test_vllm_factory_adapter.py +++ b/tests/tools/test_vllm_factory_adapter.py @@ -4,6 +4,12 @@ from __future__ import annotations +import asyncio +import json +from types import SimpleNamespace + +import pytest + from inference_service_compiler import vllm_factory_adapter as adapter @@ -35,13 +41,109 @@ def test_parse_detection_request_accepts_label_free_health_check() -> None: request = adapter.parse_detection_request( { "model": "nvidia/gliner-pii", - "messages": [{"role": "user", "content": "health check"}], + "messages": [{"role": "user", "content": "x" * 257}], + "chunk_length": 1, + "overlap": 0, } ) assert request.labels == () +def test_parse_detection_request_rejects_excessive_chunk_fanout() -> None: + """One detector request cannot enqueue unbounded pooling work.""" + + with pytest.raises(ValueError, match="at most 256 chunks"): + adapter.parse_detection_request( + { + "model": "nvidia/gliner-pii", + "messages": [{"role": "user", "content": "x" * 257}], + "labels": ["person"], + "chunk_length": 1, + "overlap": 0, + } + ) + + +def test_parse_detection_request_accepts_chunk_budget_boundary() -> None: + """The admission cap includes requests with exactly 256 chunks.""" + + request = adapter.parse_detection_request( + { + "model": "nvidia/gliner-pii", + "messages": [{"role": "user", "content": "x" * 256}], + "labels": ["person"], + "chunk_length": 1, + "overlap": 0, + } + ) + + assert request.text == "x" * 256 + + +def test_chat_compatibility_bounds_pooling_concurrency(monkeypatch: pytest.MonkeyPatch) -> None: + """Accepted chunks use a bounded worker frontier and preserve result order.""" + + async def exercise() -> None: + expected_peak = 8 + active = 0 + max_active = 0 + total_calls = 0 + reached_limit = asyncio.Event() + release = asyncio.Event() + + async def fake_invoke_pooling(**kwargs: object) -> object: + nonlocal active, max_active, total_calls + active += 1 + total_calls += 1 + max_active = max(max_active, active) + if active == expected_peak: + reached_limit.set() + try: + await release.wait() + text = kwargs["text"] + assert isinstance(text, str) + return [{"text": text, "label": "token", "start": 0, "end": 1, "score": 0.9}] + finally: + active -= 1 + + async def request_json() -> dict[str, object]: + return { + "model": "nvidia/gliner-pii", + "messages": [{"role": "user", "content": "abcdefghijklmnopq"}], + "labels": ["token"], + "chunk_length": 1, + "overlap": 0, + } + + async def call_next(_: object) -> None: + raise AssertionError("detector requests must not reach the next middleware") + + monkeypatch.setattr(adapter, "invoke_pooling", fake_invoke_pooling) + monkeypatch.setenv("ANONYMIZER_VLLM_FACTORY_PLUGIN", "deberta_gliner") + request = SimpleNamespace( + url=SimpleNamespace(path="/v1/chat/completions"), + app=SimpleNamespace(state=SimpleNamespace(serving_pooling=object())), + json=request_json, + ) + + operation = asyncio.create_task(adapter.anonymizer_chat_compatibility(request, call_next)) + await reached_limit.wait() + await asyncio.sleep(0) + observed_peak = max_active + release.set() + response = await operation + + assert response.status_code == 200 + assert observed_peak <= expected_peak + assert total_calls == 17 + payload = json.loads(response.body) + content = json.loads(payload["choices"][0]["message"]["content"]) + assert [entity["start"] for entity in content["entities"]] == list(range(17)) + + asyncio.run(exercise()) + + def test_merge_gliner_entities_restores_offsets_and_deduplicates_overlap() -> None: """Chunk-relative factory spans become stable document offsets.""" chunks = [adapter.TextChunk("Alice met Bob", 0), adapter.TextChunk("Bob at NVIDIA", 10)] diff --git a/tools/inference_service_compiler/vllm_factory_adapter.py b/tools/inference_service_compiler/vllm_factory_adapter.py index a4963d07..5a749990 100644 --- a/tools/inference_service_compiler/vllm_factory_adapter.py +++ b/tools/inference_service_compiler/vllm_factory_adapter.py @@ -18,6 +18,8 @@ DEFAULT_CHUNK_LENGTH = 384 DEFAULT_OVERLAP = 128 +MAX_CHUNKS_PER_REQUEST = 256 +MAX_CONCURRENT_POOLING_CALLS_PER_REQUEST = 8 @dataclass(frozen=True, slots=True) @@ -79,19 +81,11 @@ async def anonymizer_chat_compatibility( handler = request.app.state.serving_pooling if handler is None: raise RuntimeError("vLLM pooling handler is unavailable") - results = await asyncio.gather( - *( - invoke_pooling( - handler=handler, - model=detection.model, - plugin=plugin, - text=chunk.text, - labels=detection.labels, - threshold=detection.threshold, - flat_ner=detection.flat_ner, - ) - for chunk in chunks - ) + results = await invoke_pooling_chunks( + handler=handler, + plugin=plugin, + detection=detection, + chunks=chunks, ) entities = merge_entities( plugin=plugin, @@ -143,9 +137,13 @@ def parse_detection_request(value: object) -> DetectionRequest: flat_ner = body.get("flat_ner", False) if not isinstance(flat_ner, bool): raise ValueError("flat_ner must be a boolean") + text = extract_text(body.get("messages")) + chunk_count = _count_text_chunks(len(text), chunk_length, overlap) + if labels_value and chunk_count > MAX_CHUNKS_PER_REQUEST: + raise ValueError(f"detector requests may contain at most {MAX_CHUNKS_PER_REQUEST} chunks") return DetectionRequest( model=model, - text=extract_text(body.get("messages")), + text=text, labels=tuple(cast(list[str], labels_value)), threshold=threshold, chunk_length=chunk_length, @@ -175,6 +173,14 @@ def extract_text(messages: object) -> str: raise ValueError("message content must be a string or list") +def _count_text_chunks(text_length: int, chunk_length: int, overlap: int) -> int: + """Calculate the number of chunks without materializing their text.""" + if text_length <= chunk_length: + return 1 + stride = chunk_length - overlap + return 1 + (text_length - chunk_length + stride - 1) // stride + + def split_text(text: str, chunk_length: int, overlap: int) -> list[TextChunk]: """Split text with the same character-offset contract as the native runtime.""" if not text: @@ -189,6 +195,34 @@ def split_text(text: str, chunk_length: int, overlap: int) -> list[TextChunk]: return chunks +async def invoke_pooling_chunks( + *, + handler: Any, + plugin: FactoryPlugin, + detection: DetectionRequest, + chunks: Sequence[TextChunk], +) -> list[object]: + """Process admitted chunks with a bounded asynchronous worker frontier.""" + results = [object() for _ in chunks] + pending = iter(enumerate(chunks)) + + async def worker() -> None: + for index, chunk in pending: + results[index] = await invoke_pooling( + handler=handler, + model=detection.model, + plugin=plugin, + text=chunk.text, + labels=detection.labels, + threshold=detection.threshold, + flat_ner=detection.flat_ner, + ) + + worker_count = min(len(chunks), MAX_CONCURRENT_POOLING_CALLS_PER_REQUEST) + await asyncio.gather(*(worker() for _ in range(worker_count))) + return results + + async def invoke_pooling( *, handler: Any, From b2d558a683c06f0d588124ee664be95d04416c01 Mon Sep 17 00:00:00 2001 From: nvskills-svc-account Date: Wed, 26 Aug 2026 21:54:04 +0000 Subject: [PATCH 19/28] Attach NVSkills validation signatures Signed-off-by: nvskills-svc-account --- skills/anonymizer/BENCHMARK.md | 145 +++++++++++++++----------- skills/anonymizer/skill-card.md | 179 +++++++++++--------------------- skills/anonymizer/skill.oms.sig | 1 + 3 files changed, 149 insertions(+), 176 deletions(-) create mode 100644 skills/anonymizer/skill.oms.sig diff --git a/skills/anonymizer/BENCHMARK.md b/skills/anonymizer/BENCHMARK.md index 027917f8..7c4d1199 100644 --- a/skills/anonymizer/BENCHMARK.md +++ b/skills/anonymizer/BENCHMARK.md @@ -1,85 +1,110 @@ - - +# Skill Benchmark: anonymizer -# Evaluation Report +> ✅ **Overall verdict: PASS — Recommended for publication** -Evaluation report for the `anonymizer` skill before publication through -NVSkills-Eval. +## Publication Recommendation -This benchmark file records the publication-ready evaluation plan and task -composition for NeMo Anonymizer. The external NVSkills-Eval run has not been -executed in this local workspace, so this branch intentionally reports no -Anonymizer scores. +Recommended for publication based on the completed evaluation evidence in this report. -## Evaluation Summary +## Evaluation Metadata - Skill: `anonymizer` -- Evaluation date: pending external `/nvskills-ci` run -- NVSkills-Eval profile: external -- Environment: external NVSkills-Eval runner -- Dataset: 6 evaluation tasks -- Attempts per task: recorded by external NVSkills-Eval after execution -- Pass threshold: recorded by external NVSkills-Eval after execution -- Overall verdict: pending external NVSkills-Eval run +- Evaluation date: 2026-08-26 +- Evaluator version: `1.3.2` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 6 evaluation tasks (4 positive, 2 negative) +- Dataset digest: `sha256:2426de4eaba6137e3e0514953becf6a0eff19516d8d63bd790df3455ff8c8b32` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `local` +- Tier 3 evidence: required for publication -## Agents Used +Tasks ran on the trusted local host; local mode is not sandboxed. -Agent-level measured results are pending the external NVSkills-Eval run. +## Execution and Provenance -## Metrics Used +- Validation status: `passed` +- Report generation: `complete` +- Evaluator version: `1.3.2` +- Git commit: `0117bc2e3e54da4244a656466526c5b1b5a559ea` +- Content type: requested `auto`, detected `skill` +- Container image: `gitlab-master.nvidia.com:5005/nvcarps/ci-group/nvcarps-ci/skillevaluator-ci:sha-0117bc2e3e54da4244a656466526c5b1b5a559ea` +- Container image digest: `not recorded` +- Tier 3: requested `true`, executed `true`, status `succeeded` -Reported benchmark dimensions: +## What This Report Answers -- Security: checks whether skill-assisted execution avoids unsafe behavior such - as secret leakage, destructive commands, or unauthorized access. -- Correctness: checks whether the agent follows the expected workflow and - produces the correct final output. -- Discoverability: checks whether the agent loads the skill when relevant and - avoids using it when irrelevant. -- Effectiveness: checks whether the agent performs measurably better with the - skill than without it. -- Efficiency: checks whether the agent uses fewer tokens and avoids redundant - work. +The three-tier evaluation checks whether the skill: -Underlying evaluation signals will be recorded from the external -NVSkills-Eval output after execution. +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. -## Test Tasks +## Results at a Glance -The benchmark dataset contains 6 evaluation tasks: +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 62% → 92% (+30 points) | 71% → 95% (+24 points) | +| Security | 100% → 100% (±0 points) | 100% → 100% (±0 points) | +| Correctness | 57% → 90% (+33 points) | 77% → 97% (+20 points) | +| Discoverability | 50% → 99% (+49 points) | 67% → 94% (+27 points) | +| Effectiveness | 53% → 78% (+26 points) | 63% → 87% (+24 points) | +| Efficiency | 50% → 92% (+42 points) | 50% → 100% (+50 points) | -- Positive tasks: 4 tasks where the skill is expected to activate. -- Negative tasks: 2 tasks where no skill is expected. -- Unlabeled tasks: 0 tasks where positive/negative intent cannot be inferred. +**How to read this table:** baseline is the same task attempted without the target skill. Uplift is `skill score - baseline score`, shown in percentage points. -Entries with `should_trigger: true` and `expected_skill: "anonymizer"` are -positive skill-activation cases. Entries with `should_trigger: false` and -`expected_skill: null` are negative activation cases. +Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 percentage points above its 47% no-skill baseline. -## Results +## Tier Status -External NVSkills-Eval execution is pending. No copied or locally inferred -Anonymizer results are reported here. +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 1 validator(s); 2 finding(s) | +| Tier 2 | Semantic deduplication | **NOT RUN** | No result was recorded | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 6 task(s) | -| Dimension | Tasks | Result | -|---|---:|---| -| Security | 6 | Pending external NVSkills-Eval run | -| Correctness | 6 | Pending external NVSkills-Eval run | -| Discoverability | 6 | Pending external NVSkills-Eval run | -| Effectiveness | 6 | Pending external NVSkills-Eval run | -| Efficiency | 6 | Pending external NVSkills-Eval run | +## Findings and Observations -## Tier 1: Static Validation Summary +
+Show detailed findings and successful checks -Local static validation is covered by this branch's validation evidence. The -external NVSkills-Eval Tier 1 result is pending the `/nvskills-ci` run. +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/anonymizer/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/anonymizer/SKILL.md`) -## Tier 2: Deduplication Summary +
-External NVSkills-Eval deduplication results are pending. +## Scoring Methodology -## Publication Recommendation +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool or skill usage? | `skill_efficiency` (100%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was found and executed. +- `skill_efficiency` (Efficiency): routing quality, workspace-aware skill reads, and productive tool use. +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. + +
+ +## Freshness -Proceed to external NVSkills-Eval and signing. Publication should depend on the -external evaluation and signing results rather than this local preparation -branch alone. +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/anonymizer/skill-card.md b/skills/anonymizer/skill-card.md index 57b5aa7c..769f8e27 100644 --- a/skills/anonymizer/skill-card.md +++ b/skills/anonymizer/skill-card.md @@ -1,139 +1,86 @@ - - +## Description:
+Use when the user wants to anonymize a text dataset, redact PII, de-identify free-text data, or rewrite text to remove sensitive or inferable identifying information. Produces a runnable Python script that calls the NeMo Anonymizer pipeline (detection → replace or rewrite).
-## Description - -Use NeMo Anonymizer through an interactive agent workflow: inspect text data, -choose Replace or Rewrite, select a replacement strategy, draft a runnable -Python script, preview before full execution, diagnose failed records first, and -configure self-hosted GLiNER when detection must stay local. - -This skill package is prepared for NVSkills publication review. External -NVSkills-Eval results are pending and no Anonymizer scores are reported in this -branch. +This skill is ready for commercial/non-commercial use.
## Owner +NVIDIA
-NVIDIA - -### License/Terms of Use - -Apache 2.0 - -## Use Case - -Developers, privacy engineers, and data practitioners using NeMo Anonymizer to -detect, replace, redact, hash, annotate, or rewrite sensitive entities in text -datasets while keeping a durable script for review and reruns. - -### Deployment Geography for Use - -Global - -## Known Risks and Mitigations - -Risk: Users may overinterpret anonymized output as a privacy guarantee. - -Mitigation: The skill instructs agents to describe Anonymizer as best-effort, -preview before full execution, inspect failed records, and call out human review -for rewrite outputs that need it. - -Risk: Agent-generated scripts may target the wrong source file, text column, or -model-provider configuration. - -Mitigation: The workflow requires data inspection, explicit user confirmation -of mode and key configuration choices, and preview execution before a full run. - -Risk: An incorrect provider or model alias may send detection requests to an -unintended endpoint. - -Mitigation: The skill directs agents to configure the local GLiNER provider -explicitly, keep the full model pool, verify the endpoint, preview, and consult -the self-hosting documentation. - -## Reference(s) - -- [Interactive workflow](references/interactive.md) -- [Choosing a Strategy](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/choosing-a-strategy/) -- [Detection](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/detection/) -- [Evaluation](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/evaluation/) -- [Models](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/models/) -- [Self-hosting GLiNER](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/self-hosting-gliner/) -- [Troubleshooting](https://nvidia-nemo.github.io/Anonymizer/dev/troubleshooting/) - -## Skill Output - -**Output Type(s):** Python scripts, shell commands, configuration guidance, -diagnostic guidance +### License/Terms of Use:
+Apache 2.0
+## Use Case:
+Developers and engineers who need to de-identify or anonymize text datasets containing PII before sharing, model training, or analytics.
-**Output Format:** A runnable Python script plus concise Markdown guidance for -previewing, diagnosing failures, and running the full pipeline +### Deployment Geography for Use:
+Global
-**Output Parameters:** Dataset path, text column, data summary, mode -(`Replace` or `Rewrite`), replacement strategy when applicable, privacy goal, -risk tolerance, entity labels, and optional model-provider paths +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Yes]
+**Credential Type(s):** [API key]
-**Other Properties Related to Output:** The generated script previews by -default, exits on failed records, optionally evaluates output with -LLM-as-judge, and leaves full dataset execution under explicit user control. +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
-## Evaluation Agents Used +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
-The external NVSkills-Eval run is pending. Agent-level measured results will be -reported from the external `/nvskills-ci` evaluation output after it runs. +## Reference(s):
+- [NeMo Anonymizer Documentation](https://nvidia-nemo.github.io/Anonymizer/)
+- [Choosing a Strategy](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/choosing-a-strategy/)
+- [Detection](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/detection/)
+- [Evaluation](https://nvidia-nemo.github.io/Anonymizer/dev/concepts/evaluation/)
+- [Troubleshooting](https://nvidia-nemo.github.io/Anonymizer/dev/troubleshooting/)
+- [Interactive Reference](references/interactive.md)
-## Evaluation Tasks -The prepared evaluation dataset contains 6 NVSkills-Eval tasks: 4 positive -activation cases and 2 negative activation cases. The positive tasks cover mode -choice, stable cross-record replacement with `Hash`, failed-record-first -diagnosis, and self-hosted GLiNER. The negative tasks cover a general privacy -explainer and repository source development. +## Skill Output:
+**Output Type(s):** [Code]
+**Output Format:** [Python script]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
-## Evaluation Metrics Used +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
-Metrics will be reported by the external NVSkills-Eval run. Expected benchmark -dimensions are: -- Security: Checks whether skill-assisted execution avoids unsafe behavior such - as secret leakage, destructive commands, or unauthorized access. -- Correctness: Checks whether the agent follows the expected workflow and - produces the correct final output. -- Discoverability: Checks whether the agent loads the skill when relevant and - avoids using it when irrelevant. -- Effectiveness: Checks whether the agent performs measurably better with the - skill than without it. -- Efficiency: Checks whether the agent uses fewer tokens and avoids redundant - work. -## Evaluation Results +## Evaluation Tasks:
+6 evaluation tasks (4 positive, 2 negative) from a curated skill-evaluator dataset snapshot.
-External NVSkills-Eval execution is pending. This publication branch does not -include local or copied Anonymizer benchmark scores. +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Checks final-answer correctness against the reference answer.
+- Discoverability: Checks whether the expected skill was found and executed when needed.
+- Effectiveness: Checks whether the skill helped complete the user's goal and followed the expected workflow.
+- Efficiency: Checks routing quality, workspace-aware skill reads, and productive tool use.
-| Dimension | Tasks | Result | -|---|---:|---| -| Security | 6 | Pending external NVSkills-Eval run | -| Correctness | 6 | Pending external NVSkills-Eval run | -| Discoverability | 6 | Pending external NVSkills-Eval run | -| Effectiveness | 6 | Pending external NVSkills-Eval run | -| Efficiency | 6 | Pending external NVSkills-Eval run | +Underlying evaluation signals used in this run:
+- `security`: Detects unsafe operations, secret leakage, and unauthorized access.
+- `accuracy`: Verifies final-answer correctness against the reference answer.
+- `skill_execution`: Verifies the expected skill was found and executed.
+- `goal_accuracy`: Verifies whether the user's goal was achieved.
+- `behavior_check`: Verifies expected workflow behavior was followed.
+- `skill_efficiency`: Verifies routing quality and productive tool use.
-## Skill Version(s) -Publication candidate from this repository branch. The released skill version -should be recorded after review, external evaluation, and signing. -## Ethical Considerations +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 62% → 92% (+30 points) | 71% → 95% (+24 points) | +| Security | 100% → 100% (±0 points) | 100% → 100% (±0 points) | +| Correctness | 57% → 90% (+33 points) | 77% → 97% (+20 points) | +| Discoverability | 50% → 99% (+49 points) | 67% → 94% (+27 points) | +| Effectiveness | 53% → 78% (+26 points) | 63% → 87% (+24 points) | +| Efficiency | 50% → 92% (+42 points) | 50% → 100% (+50 points) | -NVIDIA believes Trustworthy AI is a shared responsibility and has established -policies and practices to enable development for a wide array of AI -applications. When downloaded or used in accordance with our terms of service, -developers should work with their internal team to ensure this skill meets -requirements for the relevant industry and use case and addresses foreseeable -product misuse. +## Skill Version(s):
+2cf0c90 (source: git SHA, committed 2026-08-20)
-(For Release on NVIDIA Platforms Only) +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
-Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns -[here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail). +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/anonymizer/skill.oms.sig b/skills/anonymizer/skill.oms.sig new file mode 100644 index 00000000..cf005971 --- /dev/null +++ b/skills/anonymizer/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiYW5vbnltaXplciIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI1MDMxY2RhMWQxNTBjMDI3YTE0YjMzN2QyOGEyYjJhNWIzMDVlZmI3ODNiMjE5ZGM3M2UwMjUwZDllYzQxOTYzIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlOTdlY2UzOGU5MDBhMTc1ZDNkMWE3M2NmZDlhOTZkODMxMGU4NzcxN2M2YjVjNTNlYzE0ZTc2MjBlMmJhYjMyIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjY5MjM0ZjAxYTY5YmIwNzFhMzJhOThiMzBmZWI0YjE3ZWM2MjE5YjMzZTA4MTcwNDljZWEwM2UyNzZiZjYxNGQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI5ODhjMTdmNWU1MWRkMTE3N2U1MWI3OGQ3OThhODNkZjhkNWEyM2M0NWUxYjdlN2U3ZDNjZjQzYjQ4YzI4ODUzIgogICAgICB9LAogICAgICB7CiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9pbnRlcmFjdGl2ZS5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZDc1NWFhNDU3NDA3ZTM5MDE1YzcxMWEwY2I4MDI4MmRlZTkyNzZhYWJiYzRhMTYzM2YxZDUzZDExMGUzM2YwOCIKICAgICAgfSwKICAgICAgewogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjhjYjE0Njk0MmYyZTMwMjY3ZGNmYjVlYzdlODc5MzEwMWM1OWFjZWUxMDAzNmJkNTRiNWZjYzQ0ODRiZjFkOTEiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMC9J+fw1XH4xIjnrwW+6UMUt77qe+mKOrYdeCv86/vwQ9pntvBPNlsmgIIT3iXrQHAIweyvAzhBe9seKUwM9s+NV2T0hQaq0l5hrfnjDtmJhKxy1J4/q4o1hoI7q/tZcpjm1","keyid":""}]}} \ No newline at end of file From 08d4018f547d4e924e02bd31bd0908427faf1df5 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 3 Sep 2026 17:17:53 +0000 Subject: [PATCH 20/28] fix(dev): validate inference process ownership Signed-off-by: Aaron Gonzales --- tests/tools/test_inference_service.py | 47 ++++++++++++++++++++- tools/inference_service_compiler/runtime.py | 35 ++++++++++++++- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index a9ae4339..d5e735b0 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -269,17 +269,60 @@ def wrong_model(request: httpx.Request) -> httpx.Response: runtime.probe_endpoint(plan, client=client) -def test_plan_integrity_and_pid_cleanup_are_enforced(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_plan_integrity_is_enforced_before_launch(tmp_path: Path) -> None: plan = compiler.compile_profile(generation(), source_revision="test") with pytest.raises(compiler.PlanIntegrityError): runtime.launch_plan( plan.model_copy(update={"source_revision": "changed"}), secret_values={}, log_directory=tmp_path ) + + +def test_status_and_stop_reject_a_reused_pid_without_signaling(monkeypatch: pytest.MonkeyPatch) -> None: + plan = compiler.compile_profile(generation(), source_revision="test") handle = models.LocalProcessHandle( external_id="1:x", pid=1, process_group_id=1, start_marker="old", stdout_path="out", stderr_path="err" ) + launch = launch_receipt(plan, handle) monkeypatch.setattr(runtime, "read_process_start_marker", lambda _pid: "new") - assert runtime.is_handle_running(handle) is False + + with pytest.raises(runtime.RuntimeEffectError) as status_error: + runtime.status_run(launch) + with ( + mock.patch.object(runtime.os, "killpg") as killpg, + pytest.raises(runtime.RuntimeEffectError) as stop_error, + ): + runtime.stop_run(launch) + + assert status_error.value.diagnostic.code == "process-identity-mismatch" + assert stop_error.value.diagnostic.code == "process-identity-mismatch" + killpg.assert_not_called() + + +def test_stop_rejects_a_mismatched_process_group_without_signaling() -> None: + plan = compiler.compile_profile(generation(), source_revision="test") + handle = models.LocalProcessHandle( + external_id="4242:100", + pid=4242, + process_group_id=4242, + start_marker="100", + stdout_path="out", + stderr_path="err", + ) + launch = launch_receipt(plan, handle) + + with ( + mock.patch.object(runtime, "read_process_start_marker", return_value="100"), + mock.patch.object(runtime, "read_process_state", return_value="S"), + mock.patch.object(runtime.os, "getpgid", return_value=9000), + mock.patch.object(runtime.os, "kill"), + mock.patch.object(runtime.os, "killpg") as killpg, + mock.patch.object(runtime.time, "monotonic", side_effect=[0.0, 1.0]), + pytest.raises(runtime.RuntimeEffectError) as exc_info, + ): + runtime.stop_run(launch) + + assert exc_info.value.diagnostic.code == "process-identity-mismatch" + killpg.assert_not_called() def test_launch_records_process_identity_and_resolves_secrets(tmp_path: Path) -> None: diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index 88754b70..8bb0d86b 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -172,12 +172,34 @@ def stop_run(launch: LaunchReceipt) -> StopReceipt: def is_handle_running(handle: LocalProcessHandle) -> bool: - """Check the external identity while guarding against Linux PID reuse.""" + """Check the recorded process identity without following a reused PID or group.""" current_marker = read_process_start_marker(handle.pid) - if current_marker != handle.start_marker: + if current_marker is None: return False + if current_marker != handle.start_marker: + raise _process_identity_mismatch( + handle, + f"PID {handle.pid} now has start marker {current_marker!r}, expected {handle.start_marker!r}", + ) if read_process_state(handle.pid) == "Z": return False + try: + current_process_group_id = os.getpgid(handle.pid) + except ProcessLookupError: + return False + except PermissionError as exc: + raise RuntimeEffectError( + RuntimeDiagnostic( + code="process-identity-unavailable", + message=f"cannot verify the process group for PID {handle.pid}: {exc}", + ) + ) from exc + if current_process_group_id != handle.process_group_id: + raise _process_identity_mismatch( + handle, + f"PID {handle.pid} now belongs to process group {current_process_group_id}, " + f"expected {handle.process_group_id}", + ) try: os.kill(handle.pid, 0) except ProcessLookupError: @@ -187,6 +209,15 @@ def is_handle_running(handle: LocalProcessHandle) -> bool: return True +def _process_identity_mismatch(handle: LocalProcessHandle, detail: str) -> RuntimeEffectError: + return RuntimeEffectError( + RuntimeDiagnostic( + code="process-identity-mismatch", + message=f"refusing to act on recorded process {handle.external_id}: {detail}", + ) + ) + + def _cleanup_handle(handle: LocalProcessHandle, timeout_seconds: float) -> bool: if not is_handle_running(handle): return True From 9222648fde8be824ff3bf9526b7ce7b239392047 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 3 Sep 2026 17:22:35 +0000 Subject: [PATCH 21/28] fix(dev): bind readiness to launched service Signed-off-by: Aaron Gonzales --- tests/tools/test_inference_service.py | 139 +++++++++++++++++- tests/tools/test_vllm_factory.py | 3 +- tools/inference_service_compiler/cli.py | 41 +++++- tools/inference_service_compiler/lifecycle.py | 39 +++++ tools/inference_service_compiler/runtime.py | 107 ++++++++++++-- .../vllm_runtime.py | 6 +- 6 files changed, 313 insertions(+), 22 deletions(-) create mode 100644 tools/inference_service_compiler/lifecycle.py diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index d5e735b0..345bbbc2 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -4,16 +4,18 @@ from __future__ import annotations +import asyncio import json import stat from pathlib import Path +from types import SimpleNamespace from unittest import mock import httpx import pytest from pydantic import ValidationError -from inference_service_compiler import cli, compiler, models, runtime +from inference_service_compiler import cli, compiler, lifecycle, models, runtime from inference_service_compiler.profiles import load_profile ROOT = Path(__file__).resolve().parents[2] @@ -348,6 +350,141 @@ def test_launch_records_process_identity_and_resolves_secrets(tmp_path: Path) -> assert receipt.handle.external_id == "4242:100" +def test_launch_requires_scoped_ownership_evidence(tmp_path: Path) -> None: + plan = compiler.compile_profile(generation(), source_revision="test") + handle = models.LocalProcessHandle( + external_id="4242:100", + pid=4242, + process_group_id=4242, + start_marker="100", + stdout_path="out", + stderr_path="err", + ) + probe = launch_receipt(plan, handle).probe + observed_environment: dict[str, str] = {} + observed_token: str | None = None + + def launch_process( + _plan: models.RunPlan, + _argv: tuple[str, ...], + environment: dict[str, str], + _log_directory: Path, + ) -> models.LocalProcessHandle: + observed_environment.update(environment) + return handle + + def wait_for_readiness( + _plan: models.RunPlan, + *, + secret_values: object, + handle: models.LocalProcessHandle, + launch_token: str | None = None, + ) -> models.CapabilityProbeReceipt: + del secret_values, handle + nonlocal observed_token + observed_token = launch_token + return probe + + with ( + mock.patch.object(runtime, "_launch_process", side_effect=launch_process), + mock.patch.object(runtime, "wait_for_readiness", side_effect=wait_for_readiness), + ): + receipt = runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) + + environment_token = observed_environment.get("ANONYMIZER_INFERENCE_LAUNCH_TOKEN") + assert environment_token + assert observed_token == environment_token + assert environment_token not in receipt.model_dump_json() + + +def test_launch_ownership_endpoint_requires_and_proves_the_scoped_token() -> None: + async def next_handler(_request: object) -> object: + return mock.sentinel.next_response + + def request(path: str, token: str | None) -> SimpleNamespace: + headers = {} if token is None else {lifecycle.LAUNCH_OWNERSHIP_HEADER: token} + return SimpleNamespace(url=SimpleNamespace(path=path), headers=headers) + + with mock.patch.dict( + runtime.os.environ, + {lifecycle.LAUNCH_TOKEN_ENVIRONMENT_VARIABLE: "launch-secret"}, + clear=True, + ): + wrong = asyncio.run(lifecycle.launch_ownership(request(lifecycle.LAUNCH_OWNERSHIP_PATH, "wrong"), next_handler)) + owned = asyncio.run( + lifecycle.launch_ownership(request(lifecycle.LAUNCH_OWNERSHIP_PATH, "launch-secret"), next_handler) + ) + passed_through = asyncio.run(lifecycle.launch_ownership(request("/v1/models", None), next_handler)) + + assert wrong.status_code == 404 + assert json.loads(owned.body) == { + lifecycle.LAUNCH_OWNERSHIP_PROOF_FIELD: lifecycle.launch_token_proof("launch-secret") + } + assert passed_through is mock.sentinel.next_response + + +def test_compatible_preexisting_endpoint_cannot_prove_launch_ownership() -> None: + plan = compiler.compile_profile(generation(), source_revision="test") + compatible_response = httpx.Response(200, json={"data": [{"id": plan.served_model_name}]}) + + with ( + mock.patch.object(runtime.httpx, "get", return_value=compatible_response), + pytest.raises(runtime.RuntimeEffectError) as exc_info, + ): + runtime._probe_launch_ownership(plan, "launch-secret") + + assert exc_info.value.diagnostic.code == "launch-ownership-not-observed" + + +@pytest.mark.parametrize("failure", [RuntimeError("unexpected readiness failure"), KeyboardInterrupt()]) +def test_post_spawn_readiness_failures_always_clean_up(tmp_path: Path, failure: BaseException) -> None: + plan = compiler.compile_profile(generation(), source_revision="test") + handle = models.LocalProcessHandle( + external_id="4242:100", + pid=4242, + process_group_id=4242, + start_marker="100", + stdout_path="out", + stderr_path="err", + ) + + with ( + mock.patch.object(runtime, "_launch_process", return_value=handle), + mock.patch.object(runtime, "wait_for_readiness", side_effect=failure), + mock.patch.object(runtime, "_cleanup_handle", return_value=True) as cleanup, + pytest.raises(type(failure)), + ): + runtime.launch_plan(plan, secret_values={}, log_directory=tmp_path) + + cleanup.assert_called_once_with(handle, plan.spec.local.shutdown_timeout_seconds) + + +def test_launch_command_stops_service_when_receipt_write_fails(tmp_path: Path) -> None: + plan = compiler.compile_profile(generation(), source_revision="test") + plan_path = tmp_path / "plan.json" + plan_path.write_text(plan.model_dump_json(), encoding="utf-8") + handle = models.LocalProcessHandle( + external_id="4242:100", + pid=4242, + process_group_id=4242, + start_marker="100", + stdout_path="out", + stderr_path="err", + ) + receipt = launch_receipt(plan, handle) + + with ( + mock.patch.object(cli, "launch_plan", return_value=receipt), + mock.patch.object(cli, "write_json", side_effect=PermissionError("read-only destination")), + mock.patch.object(cli, "stop_run") as stop, + pytest.raises(SystemExit) as exc_info, + ): + cli.launch(plan=plan_path, output=tmp_path / "receipt.json", log_directory=tmp_path) + + assert exc_info.value.code == 125 + stop.assert_called_once_with(receipt) + + def test_launch_refuses_an_unmarked_process_and_cleans_up(tmp_path: Path) -> None: plan = compiler.compile_profile(generation(), source_revision="test") process = mock.Mock(pid=4242) diff --git a/tests/tools/test_vllm_factory.py b/tests/tools/test_vllm_factory.py index a9359928..38d9e205 100644 --- a/tests/tools/test_vllm_factory.py +++ b/tests/tools/test_vllm_factory.py @@ -127,6 +127,7 @@ def test_factory_constructs_vllm_frontend_and_async_engine_arguments() -> None: assert arguments.enforce_eager is True assert arguments.enable_prefix_caching is True assert arguments.async_scheduling is True + assert arguments.middleware == [factory.LAUNCH_OWNERSHIP_MIDDLEWARE] assert arguments.mamba_backend.value == "flashinfer" assert arguments.mamba_ssm_cache_dtype == "float16" assert arguments.enable_mamba_cache_stochastic_rounding is True @@ -153,7 +154,7 @@ def test_factory_constructs_pooling_server_for_external_gliner_plugin() -> None: assert arguments.trust_remote_code is True assert arguments.enable_prefix_caching is False assert arguments.enable_chunked_prefill is False - assert arguments.middleware == [factory.ANONYMIZER_CHAT_MIDDLEWARE] + assert arguments.middleware == [factory.LAUNCH_OWNERSHIP_MIDDLEWARE, factory.ANONYMIZER_CHAT_MIDDLEWARE] def test_run_server_uses_the_vllm_0_27_lifecycle_boundary() -> None: diff --git a/tools/inference_service_compiler/cli.py b/tools/inference_service_compiler/cli.py index 5305804c..b69bfb3b 100644 --- a/tools/inference_service_compiler/cli.py +++ b/tools/inference_service_compiler/cli.py @@ -7,6 +7,7 @@ import functools import os import sys +import tempfile from collections.abc import Callable from pathlib import Path from tomllib import TOMLDecodeError @@ -80,12 +81,18 @@ def launch( log_directory: Path = Path(".inference-service-runs"), ) -> None: """Launch a compiled plan and record its observed handle and readiness.""" + _validate_output_destination(output) parsed = load_plan(plan.read_text(encoding="utf-8")) secret_values = {name: os.environ[name] for name in parsed.command.secret_sources if name in os.environ} - write_json( - launch_plan(parsed, secret_values=secret_values, log_directory=log_directory), - output, - ) + receipt = launch_plan(parsed, secret_values=secret_values, log_directory=log_directory) + try: + write_json(receipt, output) + except BaseException as exc: + try: + stop_run(receipt) + except BaseException as cleanup_exc: + exc.add_note(f"managed-process cleanup failed after receipt write failure: {cleanup_exc}") + raise @app.command @@ -120,4 +127,28 @@ def write_json(value: BaseModel, output: Path | None) -> None: if output is None: sys.stdout.write(rendered) else: - output.write_text(rendered, encoding="utf-8") + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + dir=output.parent, + prefix=f".{output.name}.", + delete=False, + ) as temporary_file: + temporary_file.write(rendered) + temporary_file.flush() + os.fsync(temporary_file.fileno()) + temporary_path = Path(temporary_file.name) + temporary_path.replace(output) + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + +def _validate_output_destination(output: Path | None) -> None: + """Verify that a receipt destination can create files before launching.""" + if output is None: + return + with tempfile.NamedTemporaryFile(dir=output.parent, prefix=f".{output.name}.", delete=True): + pass diff --git a/tools/inference_service_compiler/lifecycle.py b/tools/inference_service_compiler/lifecycle.py new file mode 100644 index 00000000..9a8b4f4f --- /dev/null +++ b/tools/inference_service_compiler/lifecycle.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Launch-ownership handshake shared by the controller and managed server.""" + +from __future__ import annotations + +import hashlib +import importlib +import os +import secrets +from collections.abc import Awaitable, Callable +from typing import Any + +LAUNCH_TOKEN_ENVIRONMENT_VARIABLE = "ANONYMIZER_INFERENCE_LAUNCH_TOKEN" +LAUNCH_OWNERSHIP_HEADER = "X-Anonymizer-Launch-Token" +LAUNCH_OWNERSHIP_PATH = "/_anonymizer/launch-ownership" +LAUNCH_OWNERSHIP_PROOF_FIELD = "launch_token_sha256" +LAUNCH_OWNERSHIP_MIDDLEWARE = "inference_service_compiler.lifecycle.launch_ownership" + + +def launch_token_proof(token: str) -> str: + """Return the non-secret proof expected from the launched server.""" + return hashlib.sha256(token.encode()).hexdigest() + + +async def launch_ownership( + request: Any, + call_next: Callable[[Any], Awaitable[Any]], +) -> Any: + """Prove that this server inherited the controller's launch-scoped token.""" + if request.url.path != LAUNCH_OWNERSHIP_PATH: + return await call_next(request) + + responses = importlib.import_module("starlette.responses") + expected = os.environ.get(LAUNCH_TOKEN_ENVIRONMENT_VARIABLE) + supplied = request.headers.get(LAUNCH_OWNERSHIP_HEADER) + if expected is None or supplied is None or not secrets.compare_digest(supplied, expected): + return responses.JSONResponse(status_code=404, content={"detail": "not found"}) + return responses.JSONResponse(content={LAUNCH_OWNERSHIP_PROOF_FIELD: launch_token_proof(expected)}) diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index 8bb0d86b..d0746536 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -6,6 +6,7 @@ import json import os +import secrets import signal import subprocess import time @@ -19,6 +20,13 @@ import httpx from inference_service_compiler.compiler import verify_plan +from inference_service_compiler.lifecycle import ( + LAUNCH_OWNERSHIP_HEADER, + LAUNCH_OWNERSHIP_PATH, + LAUNCH_OWNERSHIP_PROOF_FIELD, + LAUNCH_TOKEN_ENVIRONMENT_VARIABLE, + launch_token_proof, +) from inference_service_compiler.models import ( Capability, CapabilityProbeReceipt, @@ -97,9 +105,11 @@ def launch_plan( argv = plan.command.render_argv() environment = os.environ.copy() environment.update(command_environment) + launch_token = secrets.token_urlsafe(32) + environment[LAUNCH_TOKEN_ENVIRONMENT_VARIABLE] = launch_token log_directory.mkdir(parents=True, exist_ok=True) handle = _launch_process(plan, argv, environment, log_directory) - probe = _probe_or_cleanup(plan, handle, secret_values) + probe = _probe_or_cleanup(plan, handle, secret_values, launch_token) return LaunchReceipt( plan_digest=plan.plan_digest, launched_at=_now(), @@ -113,25 +123,60 @@ def _probe_or_cleanup( plan: RunPlan, handle: LocalProcessHandle, secret_values: Mapping[str, str], + launch_token: str, ) -> CapabilityProbeReceipt: try: - probe = wait_for_readiness(plan, secret_values=secret_values, handle=handle) - except RuntimeEffectError as exc: - cleanup_complete = _cleanup_handle(handle, plan.spec.local.shutdown_timeout_seconds) - raise RuntimeEffectError( - exc.diagnostic.model_copy( - update={ - "known_effects": (handle.external_id,), - "cleanup_complete": cleanup_complete, - } - ) - ) from exc + probe = wait_for_readiness( + plan, + secret_values=secret_values, + handle=handle, + launch_token=launch_token, + ) + except BaseException as exc: + cleanup_complete, cleanup_error = _attempt_failed_launch_cleanup( + handle, + plan.spec.local.shutdown_timeout_seconds, + ) + if isinstance(exc, RuntimeEffectError): + message = exc.diagnostic.message + if cleanup_error is not None: + message = f"{message}; cleanup failed: {cleanup_error}" + raise RuntimeEffectError( + exc.diagnostic.model_copy( + update={ + "message": message, + "known_effects": (handle.external_id,), + "cleanup_complete": cleanup_complete, + } + ) + ) from exc + if isinstance(exc, Exception): + message = f"readiness failed unexpectedly: {exc}" + if cleanup_error is not None: + message = f"{message}; cleanup failed: {cleanup_error}" + raise RuntimeEffectError( + RuntimeDiagnostic( + code="unexpected-readiness-failure", + message=message, + known_effects=(handle.external_id,), + cleanup_complete=cleanup_complete, + ) + ) from exc + if cleanup_error is not None: + exc.add_note(f"managed-process cleanup failed: {cleanup_error}") + raise if not probe.passed: - cleanup_complete = _cleanup_handle(handle, plan.spec.local.shutdown_timeout_seconds) + cleanup_complete, cleanup_error = _attempt_failed_launch_cleanup( + handle, + plan.spec.local.shutdown_timeout_seconds, + ) + message = "endpoint became ready but did not satisfy required capabilities" + if cleanup_error is not None: + message = f"{message}; cleanup failed: {cleanup_error}" raise RuntimeEffectError( RuntimeDiagnostic( code="capability-mismatch", - message="endpoint became ready but did not satisfy required capabilities", + message=message, known_effects=(handle.external_id,), cleanup_complete=cleanup_complete, ) @@ -139,6 +184,13 @@ def _probe_or_cleanup( return probe +def _attempt_failed_launch_cleanup(handle: LocalProcessHandle, timeout_seconds: float) -> tuple[bool, str | None]: + try: + return _cleanup_handle(handle, timeout_seconds), None + except Exception as exc: + return False, str(exc) + + def status_run(launch: LaunchReceipt) -> StatusReceipt: """Observe a recorded handle without changing process state.""" state = "running" if is_handle_running(launch.handle) else "stopped" @@ -255,6 +307,7 @@ def wait_for_readiness( *, secret_values: Mapping[str, str] | None = None, handle: LocalProcessHandle | None = None, + launch_token: str | None = None, ) -> CapabilityProbeReceipt: """Poll the declared readiness contract until it passes or times out.""" deadline = time.monotonic() + plan.readiness.timeout_seconds @@ -269,6 +322,8 @@ def wait_for_readiness( ) ) try: + if launch_token is not None: + _probe_launch_ownership(plan, launch_token) receipt = probe_endpoint(plan, secret_values=secret_values) if receipt.passed: return receipt @@ -282,6 +337,30 @@ def wait_for_readiness( raise RuntimeEffectError(RuntimeDiagnostic(code="readiness-timeout", message=message)) +def _probe_launch_ownership(plan: RunPlan, launch_token: str) -> None: + """Require a proof that the responding server inherited this launch's token.""" + url = f"{plan.endpoint.scheme}://{plan.endpoint.host}:{plan.endpoint.port}{LAUNCH_OWNERSHIP_PATH}" + try: + response = httpx.get( + url, + headers={LAUNCH_OWNERSHIP_HEADER: launch_token}, + timeout=10, + ) + if response.status_code != 200: + raise ValueError(f"launch ownership probe returned status {response.status_code}") + payload = response.json() + if not isinstance(payload, Mapping): + raise TypeError("launch ownership response must be an object") + observed_proof = payload.get(LAUNCH_OWNERSHIP_PROOF_FIELD) + expected_proof = launch_token_proof(launch_token) + if not isinstance(observed_proof, str) or not secrets.compare_digest(observed_proof, expected_proof): + raise ValueError("launch ownership proof did not match") + except (httpx.HTTPError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise RuntimeEffectError( + RuntimeDiagnostic(code="launch-ownership-not-observed", message=f"launch ownership probe failed: {exc}") + ) from exc + + def read_process_start_marker(pid: int) -> str | None: """Read Linux process start ticks to disambiguate PID reuse when available.""" stat = _read_process_stat(pid) diff --git a/tools/inference_service_compiler/vllm_runtime.py b/tools/inference_service_compiler/vllm_runtime.py index 79346180..f4c29711 100644 --- a/tools/inference_service_compiler/vllm_runtime.py +++ b/tools/inference_service_compiler/vllm_runtime.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import Literal +from inference_service_compiler.lifecycle import LAUNCH_OWNERSHIP_MIDDLEWARE from inference_service_compiler.models import FactoryPlugin, parse_factory_plugin from inference_service_compiler.vllm_factory_integration import ( io_processor_for, @@ -151,11 +152,14 @@ def build_server_arguments(parameters: VllmServerParameters) -> Namespace: ) if parameters.max_model_len is not None: engine.max_model_len = parameters.max_model_len + middleware = [LAUNCH_OWNERSHIP_MIDDLEWARE] + if factory_plugin is not None: + middleware.append(ANONYMIZER_CHAT_MIDDLEWARE) frontend = cli_args.FrontendArgs( host=parameters.host, port=parameters.port, lora_modules=lora_modules, - middleware=[ANONYMIZER_CHAT_MIDDLEWARE] if factory_plugin is not None else [], + middleware=middleware, ) values = vars(engine) | vars(frontend) values.update( From b289e2eefbbf08618240cb739088ea81e63fa28a Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 3 Sep 2026 17:24:19 +0000 Subject: [PATCH 22/28] fix(dev): bound aggregate pooling concurrency Signed-off-by: Aaron Gonzales --- tests/tools/test_vllm_factory_adapter.py | 110 ++++++++++++++---- .../vllm_factory_adapter.py | 36 ++++-- 2 files changed, 110 insertions(+), 36 deletions(-) diff --git a/tests/tools/test_vllm_factory_adapter.py b/tests/tools/test_vllm_factory_adapter.py index 7ea3b627..40d3e9e7 100644 --- a/tests/tools/test_vllm_factory_adapter.py +++ b/tests/tools/test_vllm_factory_adapter.py @@ -81,8 +81,8 @@ def test_parse_detection_request_accepts_chunk_budget_boundary() -> None: assert request.text == "x" * 256 -def test_chat_compatibility_bounds_pooling_concurrency(monkeypatch: pytest.MonkeyPatch) -> None: - """Accepted chunks use a bounded worker frontier and preserve result order.""" +def test_chat_compatibility_bounds_aggregate_pooling_concurrency(monkeypatch: pytest.MonkeyPatch) -> None: + """Concurrent requests share one worker budget and preserve result order.""" async def exercise() -> None: expected_peak = 8 @@ -90,6 +90,7 @@ async def exercise() -> None: max_active = 0 total_calls = 0 reached_limit = asyncio.Event() + exceeded_limit = asyncio.Event() release = asyncio.Event() async def fake_invoke_pooling(**kwargs: object) -> object: @@ -99,6 +100,8 @@ async def fake_invoke_pooling(**kwargs: object) -> object: max_active = max(max_active, active) if active == expected_peak: reached_limit.set() + if active > expected_peak: + exceeded_limit.set() try: await release.wait() text = kwargs["text"] @@ -107,39 +110,98 @@ async def fake_invoke_pooling(**kwargs: object) -> object: finally: active -= 1 - async def request_json() -> dict[str, object]: - return { - "model": "nvidia/gliner-pii", - "messages": [{"role": "user", "content": "abcdefghijklmnopq"}], - "labels": ["token"], - "chunk_length": 1, - "overlap": 0, - } - async def call_next(_: object) -> None: raise AssertionError("detector requests must not reach the next middleware") + def request(text: str, app_state: SimpleNamespace) -> SimpleNamespace: + async def request_json() -> dict[str, object]: + return { + "model": "nvidia/gliner-pii", + "messages": [{"role": "user", "content": text}], + "labels": ["token"], + "chunk_length": 1, + "overlap": 0, + } + + return SimpleNamespace( + url=SimpleNamespace(path="/v1/chat/completions"), + app=SimpleNamespace(state=app_state), + json=request_json, + ) + monkeypatch.setattr(adapter, "invoke_pooling", fake_invoke_pooling) monkeypatch.setenv("ANONYMIZER_VLLM_FACTORY_PLUGIN", "deberta_gliner") - request = SimpleNamespace( - url=SimpleNamespace(path="/v1/chat/completions"), - app=SimpleNamespace(state=SimpleNamespace(serving_pooling=object())), - json=request_json, - ) + app_state = SimpleNamespace(serving_pooling=object()) + requests = [request("abcdefghijklmnopq", app_state), request("ABCDEFGHIJKLMNOPQ", app_state)] - operation = asyncio.create_task(adapter.anonymizer_chat_compatibility(request, call_next)) + operations = [ + asyncio.create_task(adapter.anonymizer_chat_compatibility(active_request, call_next)) + for active_request in requests + ] await reached_limit.wait() - await asyncio.sleep(0) + try: + await asyncio.wait_for(exceeded_limit.wait(), timeout=0.05) + except TimeoutError: + pass observed_peak = max_active release.set() - response = await operation + responses = await asyncio.gather(*operations) - assert response.status_code == 200 + assert all(response.status_code == 200 for response in responses) assert observed_peak <= expected_peak - assert total_calls == 17 - payload = json.loads(response.body) - content = json.loads(payload["choices"][0]["message"]["content"]) - assert [entity["start"] for entity in content["entities"]] == list(range(17)) + assert total_calls == 34 + for response in responses: + payload = json.loads(response.body) + content = json.loads(payload["choices"][0]["message"]["content"]) + assert [entity["start"] for entity in content["entities"]] == list(range(17)) + + asyncio.run(exercise()) + + +def test_pooling_budget_is_released_after_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """A failed pooling call cannot consume a service permit permanently.""" + + async def exercise() -> None: + detection = adapter.parse_detection_request( + { + "model": "nvidia/gliner-pii", + "messages": [{"role": "user", "content": "x"}], + "labels": ["token"], + "chunk_length": 1, + "overlap": 0, + } + ) + chunks = [adapter.TextChunk("x", 0)] + limiter = asyncio.Semaphore(1) + + async def fail(**_kwargs: object) -> object: + raise RuntimeError("pooling failed") + + monkeypatch.setattr(adapter, "invoke_pooling", fail) + with pytest.raises(RuntimeError, match="pooling failed"): + await adapter.invoke_pooling_chunks( + handler=object(), + plugin="deberta_gliner", + detection=detection, + chunks=chunks, + limiter=limiter, + ) + + async def succeed(**_kwargs: object) -> object: + return [] + + monkeypatch.setattr(adapter, "invoke_pooling", succeed) + result = await asyncio.wait_for( + adapter.invoke_pooling_chunks( + handler=object(), + plugin="deberta_gliner", + detection=detection, + chunks=chunks, + limiter=limiter, + ), + timeout=0.1, + ) + assert result == [[]] asyncio.run(exercise()) diff --git a/tools/inference_service_compiler/vllm_factory_adapter.py b/tools/inference_service_compiler/vllm_factory_adapter.py index 5a749990..cb991418 100644 --- a/tools/inference_service_compiler/vllm_factory_adapter.py +++ b/tools/inference_service_compiler/vllm_factory_adapter.py @@ -19,7 +19,8 @@ DEFAULT_CHUNK_LENGTH = 384 DEFAULT_OVERLAP = 128 MAX_CHUNKS_PER_REQUEST = 256 -MAX_CONCURRENT_POOLING_CALLS_PER_REQUEST = 8 +MAX_CONCURRENT_POOLING_CALLS = 8 +POOLING_LIMITER_STATE_ATTRIBUTE = "_anonymizer_pooling_limiter" @dataclass(frozen=True, slots=True) @@ -86,6 +87,7 @@ async def anonymizer_chat_compatibility( plugin=plugin, detection=detection, chunks=chunks, + limiter=_pooling_limiter(request.app.state), ) entities = merge_entities( plugin=plugin, @@ -201,28 +203,38 @@ async def invoke_pooling_chunks( plugin: FactoryPlugin, detection: DetectionRequest, chunks: Sequence[TextChunk], + limiter: asyncio.Semaphore, ) -> list[object]: - """Process admitted chunks with a bounded asynchronous worker frontier.""" + """Process admitted chunks within the server-wide pooling budget.""" results = [object() for _ in chunks] pending = iter(enumerate(chunks)) async def worker() -> None: for index, chunk in pending: - results[index] = await invoke_pooling( - handler=handler, - model=detection.model, - plugin=plugin, - text=chunk.text, - labels=detection.labels, - threshold=detection.threshold, - flat_ner=detection.flat_ner, - ) + async with limiter: + results[index] = await invoke_pooling( + handler=handler, + model=detection.model, + plugin=plugin, + text=chunk.text, + labels=detection.labels, + threshold=detection.threshold, + flat_ner=detection.flat_ner, + ) - worker_count = min(len(chunks), MAX_CONCURRENT_POOLING_CALLS_PER_REQUEST) + worker_count = min(len(chunks), MAX_CONCURRENT_POOLING_CALLS) await asyncio.gather(*(worker() for _ in range(worker_count))) return results +def _pooling_limiter(app_state: Any) -> asyncio.Semaphore: + limiter = getattr(app_state, POOLING_LIMITER_STATE_ATTRIBUTE, None) + if limiter is None: + limiter = asyncio.Semaphore(MAX_CONCURRENT_POOLING_CALLS) + setattr(app_state, POOLING_LIMITER_STATE_ATTRIBUTE, limiter) + return cast(asyncio.Semaphore, limiter) + + async def invoke_pooling( *, handler: Any, From c33a1e398d118fa1bcb01753908f0a1e3f83d533 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 3 Sep 2026 17:27:18 +0000 Subject: [PATCH 23/28] refactor(dev): isolate readiness failure cleanup Signed-off-by: Aaron Gonzales --- tools/inference_service_compiler/runtime.py | 66 ++++++++++----------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index d0746536..172df041 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -15,7 +15,7 @@ from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path -from typing import Literal, assert_never, cast +from typing import Literal, Never, assert_never, cast import httpx @@ -133,38 +133,7 @@ def _probe_or_cleanup( launch_token=launch_token, ) except BaseException as exc: - cleanup_complete, cleanup_error = _attempt_failed_launch_cleanup( - handle, - plan.spec.local.shutdown_timeout_seconds, - ) - if isinstance(exc, RuntimeEffectError): - message = exc.diagnostic.message - if cleanup_error is not None: - message = f"{message}; cleanup failed: {cleanup_error}" - raise RuntimeEffectError( - exc.diagnostic.model_copy( - update={ - "message": message, - "known_effects": (handle.external_id,), - "cleanup_complete": cleanup_complete, - } - ) - ) from exc - if isinstance(exc, Exception): - message = f"readiness failed unexpectedly: {exc}" - if cleanup_error is not None: - message = f"{message}; cleanup failed: {cleanup_error}" - raise RuntimeEffectError( - RuntimeDiagnostic( - code="unexpected-readiness-failure", - message=message, - known_effects=(handle.external_id,), - cleanup_complete=cleanup_complete, - ) - ) from exc - if cleanup_error is not None: - exc.add_note(f"managed-process cleanup failed: {cleanup_error}") - raise + _raise_readiness_failure(exc, handle, plan.spec.local.shutdown_timeout_seconds) if not probe.passed: cleanup_complete, cleanup_error = _attempt_failed_launch_cleanup( handle, @@ -191,6 +160,37 @@ def _attempt_failed_launch_cleanup(handle: LocalProcessHandle, timeout_seconds: return False, str(exc) +def _raise_readiness_failure( + exc: BaseException, + handle: LocalProcessHandle, + timeout_seconds: float, +) -> Never: + cleanup_complete, cleanup_error = _attempt_failed_launch_cleanup(handle, timeout_seconds) + cleanup_suffix = f"; cleanup failed: {cleanup_error}" if cleanup_error is not None else "" + if isinstance(exc, RuntimeEffectError): + raise RuntimeEffectError( + exc.diagnostic.model_copy( + update={ + "message": f"{exc.diagnostic.message}{cleanup_suffix}", + "known_effects": (handle.external_id,), + "cleanup_complete": cleanup_complete, + } + ) + ) from exc + if isinstance(exc, Exception): + raise RuntimeEffectError( + RuntimeDiagnostic( + code="unexpected-readiness-failure", + message=f"readiness failed unexpectedly: {exc}{cleanup_suffix}", + known_effects=(handle.external_id,), + cleanup_complete=cleanup_complete, + ) + ) from exc + if cleanup_error is not None: + exc.add_note(f"managed-process cleanup failed: {cleanup_error}") + raise exc + + def status_run(launch: LaunchReceipt) -> StatusReceipt: """Observe a recorded handle without changing process state.""" state = "running" if is_handle_running(launch.handle) else "stopped" From 050f75c53b53b2c83f69141b8a6785bdb31e46f0 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 3 Sep 2026 17:27:35 +0000 Subject: [PATCH 24/28] docs: describe managed inference safety guarantees Signed-off-by: Aaron Gonzales --- docs/concepts/inference-services.md | 28 +++++++++++++++++----------- docs/concepts/self-hosting-gliner.md | 13 +++++++------ 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/docs/concepts/inference-services.md b/docs/concepts/inference-services.md index 07c00303..147b9d60 100644 --- a/docs/concepts/inference-services.md +++ b/docs/concepts/inference-services.md @@ -61,10 +61,13 @@ This developer tool trusts whoever can write the profile and plan: it does not authenticate the plan's author, sign the plan, or recompile the embedded spec to prove that every derived field is semantically consistent. -`launch` waits for `/v1/models` and a task-specific request. A generation -service must return a chat completion. A detector must demonstrate dynamic -labels, offsets, and scores. The launch receipt records the process group and -Linux start marker so later commands do not signal a reused PID. +`launch` first requires a launch-scoped ownership proof from the new process, +then waits for `/v1/models` and a task-specific request. The ownership check +prevents an existing compatible service on the target port from satisfying a +new launch. A generation service must return a chat completion. A detector +must demonstrate dynamic labels, offsets, and scores. The launch receipt +records the process group and Linux start marker so later commands do not +signal a reused PID or an unrelated process group. The v2 profile schema has four sections: @@ -106,10 +109,13 @@ uv run --python 3.12 python tools/inference_service.py launch \ --log-directory .inference-service-runs ``` -The command returns after both readiness checks pass. Keep `gliner-launch.json`: -it is an unsigned durable operation record containing the observed handle and -its consistency fingerprint. It does not own or prove process identity; later -status and stop operations re-check the exact PID and start marker. +The command returns after the ownership and readiness checks pass. Keep +`gliner-launch.json`: it is an unsigned durable operation record containing the +observed handle and its consistency fingerprint. The tool validates the output +destination before launch and publishes file receipts with atomic replacement. +If publication still fails, it makes a bounded attempt to stop the launched +process. Later status and stop operations re-check the exact PID, Linux start +marker, and process group. ### Operate the service @@ -126,9 +132,9 @@ uv run --python 3.12 python tools/inference_service.py stop \ --receipt gliner-launch.json ``` -`stop` checks the recorded PID and start marker before sending `SIGTERM` to its -process group, waits for the profile's shutdown timeout, and uses `SIGKILL` if -the group remains alive. +`stop` checks the recorded PID, start marker, and live process group before +sending `SIGTERM`, waits for the profile's shutdown timeout, and uses `SIGKILL` +if the process remains alive. ## Deploy in a GPU container diff --git a/docs/concepts/self-hosting-gliner.md b/docs/concepts/self-hosting-gliner.md index b5532936..8ebc0ce7 100644 --- a/docs/concepts/self-hosting-gliner.md +++ b/docs/concepts/self-hosting-gliner.md @@ -84,9 +84,9 @@ entities = merge_entities(plugin=plugin, chunks=chunks, results=results, ...) ``` The adapter rejects a labeled request that would produce more than 256 chunks -before it materializes chunk text or calls vLLM. Accepted requests run at most -eight pooling calls concurrently; the remaining admitted chunks wait for a -worker. +before it materializes chunk text or calls vLLM. The service runs at most eight +pooling calls concurrently across all accepted requests; the remaining +admitted chunks wait for capacity. When `flat_ner` is `false` (Anonymizer's default), the adapter removes nested subset spans before score-based deduplication across chunk overlaps. A request @@ -132,9 +132,10 @@ plan as shown in [Deploy local models](inference-services.md). That guide has complete host and GPU container workflows. NVIDIA GLiNER uses `deberta_gliner`; GLiNER2 uses `deberta_gliner2`. -Launch writes a versioned receipt only after the model-list and detection -contract probes pass. Use that receipt with the compiler's `status` and -`stop` commands instead of supervising the internal server module directly. +Launch writes a versioned receipt only after the new process proves launch +ownership and the model-list and detection contract probes pass. Use that +receipt with the compiler's `status` and `stop` commands instead of supervising +the internal server module directly. The model families do not use identical label vocabularies. The request example below targets the default NVIDIA model and uses `user_name`; the default GLiNER2 PII checkpoint uses `username` for that category. From d1842ad6644f8c2b6a81d1d89c7496c73b6ed526 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 3 Sep 2026 17:28:41 +0000 Subject: [PATCH 25/28] fix(dev): reject invalid receipt targets early Signed-off-by: Aaron Gonzales --- tests/tools/test_inference_service.py | 27 +++++++++++++++++++++++++ tools/inference_service_compiler/cli.py | 2 ++ 2 files changed, 29 insertions(+) diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index 345bbbc2..76703944 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -485,6 +485,33 @@ def test_launch_command_stops_service_when_receipt_write_fails(tmp_path: Path) - stop.assert_called_once_with(receipt) +def test_launch_command_rejects_a_directory_receipt_target_before_launch(tmp_path: Path) -> None: + plan = compiler.compile_profile(generation(), source_revision="test") + plan_path = tmp_path / "plan.json" + plan_path.write_text(plan.model_dump_json(), encoding="utf-8") + output = tmp_path / "receipt" + output.mkdir() + handle = models.LocalProcessHandle( + external_id="4242:100", + pid=4242, + process_group_id=4242, + start_marker="100", + stdout_path="out", + stderr_path="err", + ) + receipt = launch_receipt(plan, handle) + + with ( + mock.patch.object(cli, "launch_plan", return_value=receipt) as launch, + mock.patch.object(cli, "stop_run"), + pytest.raises(SystemExit) as exc_info, + ): + cli.launch(plan=plan_path, output=output, log_directory=tmp_path) + + assert exc_info.value.code == 125 + launch.assert_not_called() + + def test_launch_refuses_an_unmarked_process_and_cleans_up(tmp_path: Path) -> None: plan = compiler.compile_profile(generation(), source_revision="test") process = mock.Mock(pid=4242) diff --git a/tools/inference_service_compiler/cli.py b/tools/inference_service_compiler/cli.py index b69bfb3b..54316a0c 100644 --- a/tools/inference_service_compiler/cli.py +++ b/tools/inference_service_compiler/cli.py @@ -150,5 +150,7 @@ def _validate_output_destination(output: Path | None) -> None: """Verify that a receipt destination can create files before launching.""" if output is None: return + if output.is_dir(): + raise IsADirectoryError(output) with tempfile.NamedTemporaryFile(dir=output.parent, prefix=f".{output.name}.", delete=True): pass From 7b4e515c592dbdf00bd0aad347bfe68c4b046aed Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 3 Sep 2026 17:29:49 +0000 Subject: [PATCH 26/28] fix(dev): authenticate launch ownership probes Signed-off-by: Aaron Gonzales --- tests/tools/test_inference_service.py | 36 +++++++++++++++++++++ tools/inference_service_compiler/runtime.py | 12 +++++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index 76703944..f55f8f5b 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -436,6 +436,42 @@ def test_compatible_preexisting_endpoint_cannot_prove_launch_ownership() -> None assert exc_info.value.diagnostic.code == "launch-ownership-not-observed" +def test_launch_ownership_probe_receives_endpoint_authentication() -> None: + plan = compiler.compile_profile(generation(api_key_env="LOCAL_KEY"), source_revision="test") + probe = launch_receipt( + plan, + models.LocalProcessHandle( + external_id="4242:100", + pid=4242, + process_group_id=4242, + start_marker="100", + stdout_path="out", + stderr_path="err", + ), + ).probe + observed_secrets: object = None + + def ownership_probe( + _plan: models.RunPlan, + _launch_token: str, + secret_values: object = None, + ) -> None: + nonlocal observed_secrets + observed_secrets = secret_values + + with ( + mock.patch.object(runtime, "_probe_launch_ownership", side_effect=ownership_probe), + mock.patch.object(runtime, "probe_endpoint", return_value=probe), + ): + runtime.wait_for_readiness( + plan, + secret_values={"LOCAL_KEY": "endpoint-secret"}, + launch_token="launch-secret", + ) + + assert observed_secrets == {"LOCAL_KEY": "endpoint-secret"} + + @pytest.mark.parametrize("failure", [RuntimeError("unexpected readiness failure"), KeyboardInterrupt()]) def test_post_spawn_readiness_failures_always_clean_up(tmp_path: Path, failure: BaseException) -> None: plan = compiler.compile_profile(generation(), source_revision="test") diff --git a/tools/inference_service_compiler/runtime.py b/tools/inference_service_compiler/runtime.py index 172df041..7b71c612 100644 --- a/tools/inference_service_compiler/runtime.py +++ b/tools/inference_service_compiler/runtime.py @@ -323,7 +323,7 @@ def wait_for_readiness( ) try: if launch_token is not None: - _probe_launch_ownership(plan, launch_token) + _probe_launch_ownership(plan, launch_token, secret_values) receipt = probe_endpoint(plan, secret_values=secret_values) if receipt.passed: return receipt @@ -337,13 +337,19 @@ def wait_for_readiness( raise RuntimeEffectError(RuntimeDiagnostic(code="readiness-timeout", message=message)) -def _probe_launch_ownership(plan: RunPlan, launch_token: str) -> None: +def _probe_launch_ownership( + plan: RunPlan, + launch_token: str, + secret_values: Mapping[str, str] | None = None, +) -> None: """Require a proof that the responding server inherited this launch's token.""" url = f"{plan.endpoint.scheme}://{plan.endpoint.host}:{plan.endpoint.port}{LAUNCH_OWNERSHIP_PATH}" + headers = _probe_headers(plan, secret_values or {}) + headers[LAUNCH_OWNERSHIP_HEADER] = launch_token try: response = httpx.get( url, - headers={LAUNCH_OWNERSHIP_HEADER: launch_token}, + headers=headers, timeout=10, ) if response.status_code != 200: From e0570b44c6f95b16625abb0606f3ace2cc6bbf2a Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Thu, 3 Sep 2026 18:24:50 +0000 Subject: [PATCH 27/28] feat(dev): add Gemma and Nemotron NVFP4 profiles Signed-off-by: Aaron Gonzales --- docs/concepts/inference-services.md | 12 +++- tests/tools/test_inference_service.py | 58 +++++++++++++++++++ .../gemma-4-12b-it.toml | 24 ++++++++ .../nemotron-3.5-lightning-nvfp4.toml | 28 +++++++++ 4 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 tools/inference_service_profiles/gemma-4-12b-it.toml create mode 100644 tools/inference_service_profiles/nemotron-3.5-lightning-nvfp4.toml diff --git a/docs/concepts/inference-services.md b/docs/concepts/inference-services.md index 147b9d60..817c82d5 100644 --- a/docs/concepts/inference-services.md +++ b/docs/concepts/inference-services.md @@ -19,23 +19,33 @@ The tool lives under `tools/` and is not installed with the ## Choose models -Seven profiles ship in `tools/inference_service_profiles/`: +Nine profiles ship in `tools/inference_service_profiles/`: | Profile | Endpoint model name | Role and hardware guidance | | --- | --- | --- | | `nvidia-gliner.toml` | `nvidia/gliner-pii` | Default PII detector; can share an 80 GB GPU with a 20B or 30B generator | | `gliner2.toml` | `fastino/gliner2-privacy-filter-PII-multi` | Alternative multilingual PII detector | | `vllm-local.toml` | `anonymizer-local` | TinyLlama lifecycle smoke test | +| `gemma-4-12b-it.toml` | `gemma-4-12b-it-local` | Instruction-tuned Gemma 4 generation; requires an NVIDIA GPU with at least 40 GB of memory | | `gpt-oss-20b.toml` | `gpt-oss-20b-local` | Compact GPT-OSS generation | | `gpt-oss-120b.toml` | `gpt-oss-120b-local` | GPT-OSS generation on a dedicated 80 GB GPU | | `qwen3-30b-a3b-instruct.toml` | `qwen3-30b-a3b-instruct-local` | Multilingual generation | | `nemotron-3.5-lightning.toml` | `nemotron-3.5-lightning-local` | High-throughput generation on an 80 GB GPU | +| `nemotron-3.5-lightning-nvfp4.toml` | `nemotron-3.5-lightning-nvfp4-local` | Official NVFP4 checkpoint with a conservative single-H100 configuration | Each profile pins its Hugging Face revision. Generation profiles use stock vLLM. The two detector profiles use the pinned external [vLLM Factory](https://github.com/latenceainew/vllm-factory) integration and Anonymizer's OpenAI-compatible detector adapter. +The Gemma profile uses the instruction-tuned +[`google/gemma-4-12B-it`](https://huggingface.co/google/gemma-4-12B-it) +checkpoint and bounds its larger native context window to 8,192 tokens. The +Nemotron NVFP4 profile uses NVIDIA's +[`NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4`](https://huggingface.co/nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4) +checkpoint. vLLM detects its ModelOpt quantization from the checkpoint, so the +profile does not request online quantization. + GPU memory also depends on the architecture, context length, concurrency, and other processes. Lower `max_model_len`, `max_num_seqs`, or `gpu_memory_utilization` when vLLM cannot reserve its KV cache. Do not co-host diff --git a/tests/tools/test_inference_service.py b/tests/tools/test_inference_service.py index f55f8f5b..adf293a2 100644 --- a/tests/tools/test_inference_service.py +++ b/tests/tools/test_inference_service.py @@ -68,6 +68,64 @@ def test_all_shipped_profiles_compile() -> None: assert all(plan.served_model_name for plan in plans) +@pytest.mark.parametrize( + ("filename", "model_id", "revision", "served_model_name", "valued_flags", "switches"), + [ + ( + "gemma-4-12b-it.toml", + "google/gemma-4-12B-it", + "707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7", + "gemma-4-12b-it-local", + (("--gpu-memory-utilization", "0.9"), ("--max-model-len", "8192"), ("--max-num-seqs", "16")), + ("--enable-prefix-caching", "--async-scheduling"), + ), + ( + "nemotron-3.5-lightning-nvfp4.toml", + "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + "cc84af2fe71647d87f4486c064f320e1e7535243", + "nemotron-3.5-lightning-nvfp4-local", + ( + ("--gpu-memory-utilization", "0.88"), + ("--max-model-len", "8192"), + ("--max-num-seqs", "16"), + ("--mamba-backend", "flashinfer"), + ("--mamba-ssm-cache-dtype", "float16"), + ("--mamba-cache-philox-rounds", "5"), + ), + ( + "--enable-prefix-caching", + "--async-scheduling", + "--enable-mamba-cache-stochastic-rounding", + ), + ), + ], +) +def test_dedicated_generation_profiles_are_pinned_and_compile_expected_flags( + filename: str, + model_id: str, + revision: str, + served_model_name: str, + valued_flags: tuple[tuple[str, str], ...], + switches: tuple[str, ...], +) -> None: + profile_path = PROFILES / filename + assert profile_path.is_file(), f"missing dedicated profile: {filename}" + + plan = compiler.compile_profile(load_profile(profile_path), source_revision="test") + argv = plan.command.render_argv() + + assert plan.spec.model.model_id == model_id + assert plan.spec.model.revision == revision + assert plan.served_model_name == served_model_name + assert plan.required_capabilities == ("chat-completions",) + assert argv[2] == model_id + for flag, expected_value in (("--revision", revision), ("--tokenizer-revision", revision), *valued_flags): + flag_index = argv.index(flag) + assert argv[flag_index + 1] == expected_value + assert all(switch in argv for switch in switches) + assert "--vllm-factory-plugin" not in argv + + def test_plan_keeps_one_endpoint_address_and_one_served_model_vocabulary() -> None: plan = compiler.compile_profile(generation(served_model_name="local-generator"), source_revision="test") diff --git a/tools/inference_service_profiles/gemma-4-12b-it.toml b/tools/inference_service_profiles/gemma-4-12b-it.toml new file mode 100644 index 00000000..ab92de3f --- /dev/null +++ b/tools/inference_service_profiles/gemma-4-12b-it.toml @@ -0,0 +1,24 @@ +schema_version = "inference-service.local-spec/v2" + +[task] +kind = "generation" + +[model] +model_id = "google/gemma-4-12B-it" +revision = "707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7" + +[vllm] +python_executable = ".venv/bin/python" +served_model_name = "gemma-4-12b-it-local" +gpu_memory_utilization = 0.90 +max_model_len = 8192 +max_num_seqs = 16 +enable_prefix_caching = true +async_scheduling = true + +[local] +host = "127.0.0.1" +port = 8000 + +startup_timeout_seconds = 1800 +shutdown_timeout_seconds = 60 diff --git a/tools/inference_service_profiles/nemotron-3.5-lightning-nvfp4.toml b/tools/inference_service_profiles/nemotron-3.5-lightning-nvfp4.toml new file mode 100644 index 00000000..09281d9a --- /dev/null +++ b/tools/inference_service_profiles/nemotron-3.5-lightning-nvfp4.toml @@ -0,0 +1,28 @@ +schema_version = "inference-service.local-spec/v2" + +[task] +kind = "generation" + +[model] +model_id = "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4" +revision = "cc84af2fe71647d87f4486c064f320e1e7535243" + +[vllm] +python_executable = ".venv/bin/python" +served_model_name = "nemotron-3.5-lightning-nvfp4-local" +gpu_memory_utilization = 0.88 +max_model_len = 8192 +max_num_seqs = 16 +enable_prefix_caching = true +async_scheduling = true +mamba_backend = "flashinfer" +mamba_ssm_cache_dtype = "float16" +enable_mamba_cache_stochastic_rounding = true +mamba_cache_philox_rounds = 5 + +[local] +host = "127.0.0.1" +port = 8000 + +startup_timeout_seconds = 1800 +shutdown_timeout_seconds = 60 From 8562bdd285019fa36f895e39b03beb4aa9eb53d2 Mon Sep 17 00:00:00 2001 From: Aaron Gonzales Date: Fri, 11 Sep 2026 15:55:23 +0000 Subject: [PATCH 28/28] fix(dev): enforce vLLM API authentication Signed-off-by: Aaron Gonzales --- tests/tools/test_vllm_factory_adapter.py | 80 +++++++++++++++++++ .../vllm_factory_adapter.py | 20 +++++ 2 files changed, 100 insertions(+) diff --git a/tests/tools/test_vllm_factory_adapter.py b/tests/tools/test_vllm_factory_adapter.py index 40d3e9e7..284115b7 100644 --- a/tests/tools/test_vllm_factory_adapter.py +++ b/tests/tools/test_vllm_factory_adapter.py @@ -9,6 +9,7 @@ from types import SimpleNamespace import pytest +from starlette.datastructures import Headers from inference_service_compiler import vllm_factory_adapter as adapter @@ -81,6 +82,85 @@ def test_parse_detection_request_accepts_chunk_budget_boundary() -> None: assert request.text == "x" * 256 +@pytest.mark.parametrize( + "authorization", + [None, "Basic expected-secret", "Bearer wrong-secret"], +) +def test_chat_compatibility_rejects_invalid_configured_api_key( + monkeypatch: pytest.MonkeyPatch, + authorization: str | None, +) -> None: + """The compatibility route preserves vLLM's configured authentication.""" + + async def exercise() -> None: + async def request_json() -> dict[str, object]: + return { + "model": "nvidia/gliner-pii", + "messages": [{"role": "user", "content": "Ada Lovelace"}], + "labels": [], + } + + async def call_next(_: object) -> None: + raise AssertionError("the compatibility route must handle this request") + + headers = Headers({} if authorization is None else {"authorization": authorization}) + request = SimpleNamespace( + url=SimpleNamespace(path="/v1/chat/completions"), + headers=headers, + json=request_json, + ) + + response = await adapter.anonymizer_chat_compatibility(request, call_next) + + assert response.status_code == 401 + assert json.loads(response.body) == {"error": "Unauthorized"} + + monkeypatch.setenv("VLLM_API_KEY", "expected-secret") + monkeypatch.setenv("ANONYMIZER_VLLM_FACTORY_PLUGIN", "deberta_gliner") + asyncio.run(exercise()) + + +@pytest.mark.parametrize( + ("api_key", "authorization"), + [(None, None), ("expected-secret", "Bearer expected-secret")], +) +def test_chat_compatibility_accepts_valid_or_disabled_api_key( + monkeypatch: pytest.MonkeyPatch, + api_key: str | None, + authorization: str | None, +) -> None: + """The compatibility route remains available without auth or with a valid key.""" + + async def exercise() -> None: + async def request_json() -> dict[str, object]: + return { + "model": "nvidia/gliner-pii", + "messages": [{"role": "user", "content": "Ada Lovelace"}], + "labels": [], + } + + async def call_next(_: object) -> None: + raise AssertionError("the compatibility route must handle this request") + + headers = Headers({} if authorization is None else {"authorization": authorization}) + request = SimpleNamespace( + url=SimpleNamespace(path="/v1/chat/completions"), + headers=headers, + json=request_json, + ) + + response = await adapter.anonymizer_chat_compatibility(request, call_next) + + assert response.status_code == 200 + + if api_key is None: + monkeypatch.delenv("VLLM_API_KEY", raising=False) + else: + monkeypatch.setenv("VLLM_API_KEY", api_key) + monkeypatch.setenv("ANONYMIZER_VLLM_FACTORY_PLUGIN", "deberta_gliner") + asyncio.run(exercise()) + + def test_chat_compatibility_bounds_aggregate_pooling_concurrency(monkeypatch: pytest.MonkeyPatch) -> None: """Concurrent requests share one worker budget and preserve result order.""" diff --git a/tools/inference_service_compiler/vllm_factory_adapter.py b/tools/inference_service_compiler/vllm_factory_adapter.py index cb991418..87ba1034 100644 --- a/tools/inference_service_compiler/vllm_factory_adapter.py +++ b/tools/inference_service_compiler/vllm_factory_adapter.py @@ -5,10 +5,12 @@ from __future__ import annotations import asyncio +import hashlib import importlib import json import math import os +import secrets import uuid from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass @@ -21,6 +23,7 @@ MAX_CHUNKS_PER_REQUEST = 256 MAX_CONCURRENT_POOLING_CALLS = 8 POOLING_LIMITER_STATE_ATTRIBUTE = "_anonymizer_pooling_limiter" +VLLM_API_KEY_ENV = "VLLM_API_KEY" @dataclass(frozen=True, slots=True) @@ -73,6 +76,8 @@ async def anonymizer_chat_compatibility( return await call_next(request) responses = importlib.import_module("starlette.responses") + if not _has_valid_api_key(request): + return responses.JSONResponse(content={"error": "Unauthorized"}, status_code=401) try: detection = parse_detection_request(await request.json()) plugin = parse_factory_plugin(os.environ["ANONYMIZER_VLLM_FACTORY_PLUGIN"]) @@ -118,6 +123,21 @@ async def anonymizer_chat_compatibility( ) +def _has_valid_api_key(request: Any) -> bool: + api_key = os.environ.get(VLLM_API_KEY_ENV) + if not api_key or getattr(request, "method", None) == "OPTIONS": + return True + authorization = request.headers.get("Authorization") + if not authorization: + return False + scheme, _, token = authorization.partition(" ") + if scheme.lower() != "bearer": + return False + expected_digest = hashlib.sha256(api_key.encode("utf-8")).digest() + actual_digest = hashlib.sha256(token.encode("utf-8")).digest() + return secrets.compare_digest(actual_digest, expected_digest) + + def parse_detection_request(value: object) -> DetectionRequest: """Validate the bounded chat-completions request used by Anonymizer.""" body = require_mapping(value, "request body")