diff --git a/Formula/omlx.rb b/Formula/omlx.rb index 45d48e6f2..38099af38 100644 --- a/Formula/omlx.rb +++ b/Formula/omlx.rb @@ -1,8 +1,8 @@ class Omlx < Formula desc "LLM inference server optimized for Apple Silicon" homepage "https://github.com/jundot/omlx" - url "https://github.com/jundot/omlx/archive/refs/tags/v0.3.5.tar.gz" - sha256 "d40f7b13a35e944f0c00fd9005e6667bd2b8be083f6cce3396f97a733fe22876" + url "https://github.com/jundot/omlx/archive/refs/tags/v0.3.6.tar.gz" + sha256 "61135fcc60ca7f9b2a9da3d6c06646963a374f9173918d484916933636ab058b" license "Apache-2.0" head "https://github.com/jundot/omlx.git", branch: "main" diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000..82f0b5ab8 --- /dev/null +++ b/conftest.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +""" +Root conftest.py — mock Apple-Silicon-only dependencies so that unit tests +for pure-Python modules (e.g. omlx.mcp.*) can run on Linux CI runners. +""" + +import sys +import types +from unittest.mock import MagicMock + + +def _mock_mlx() -> None: + """Insert MagicMock stubs for mlx and mlx_lm before any imports occur.""" + if "mlx.core" in sys.modules: + # Already importable (macOS with MLX installed) — nothing to do. + try: + import mlx.core # noqa: F401 + return + except ImportError: + pass + + # Build a minimal package tree that satisfies all sub-module imports + # encountered in omlx/scheduler.py and friends. + for pkg_name in ("mlx", "mlx_lm", "mlx_embeddings", "mlx_vlm"): + pkg = types.ModuleType(pkg_name) + pkg.__path__ = [] # mark as package + sys.modules[pkg_name] = pkg + + # Attach commonly referenced sub-modules + _submodules = [ + "mlx.core", + "mlx.nn", + "mlx.nn.layers", + "mlx.optimizers", + "mlx.utils", + "mlx_lm.generate", + "mlx_lm.utils", + "mlx_lm.models", + "mlx_lm.models.base", + "mlx_lm.models.cache", + "mlx_lm.sample_utils", + "mlx_lm.tokenizer_utils", + "mlx_embeddings.core", + "mlx_vlm.utils", + ] + for name in _submodules: + if name not in sys.modules: + sys.modules[name] = MagicMock() + + # Make mlx.core attributes look real enough for isinstance checks + mx = sys.modules["mlx.core"] + mx.array = MagicMock # type: ignore[attr-defined] + + +_mock_mlx() diff --git a/mcp.example.json b/mcp.example.json index 76c326ef5..f6731cfef 100644 --- a/mcp.example.json +++ b/mcp.example.json @@ -19,6 +19,30 @@ "url": "http://localhost:3001/sse", "enabled": false, "timeout": 60 + }, + "notion": { + "_comment": "Notion remote MCP — uses OAuth 2.1 with Dynamic Client Registration (no client_id needed). Run: omlx mcp login notion", + "transport": "streamable-http", + "url": "https://mcp.notion.com/mcp", + "enabled": false, + "timeout": 60, + "auth": { + "type": "oauth2" + } + }, + "notion-oauth-app": { + "_comment": "Notion public OAuth app — for developers with a registered OAuth application (client_id required)", + "transport": "streamable-http", + "url": "https://mcp.notion.com/mcp", + "enabled": false, + "timeout": 60, + "auth": { + "type": "oauth2", + "client_id": "YOUR_NOTION_OAUTH_CLIENT_ID", + "auth_url": "https://api.notion.com/v1/oauth/authorize", + "token_url": "https://api.notion.com/v1/oauth/token", + "scopes": ["read_content", "update_content"] + } } }, "max_tool_calls": 10, diff --git a/omlx/_version.py b/omlx/_version.py index a8d4557d2..d7b30e121 100644 --- a/omlx/_version.py +++ b/omlx/_version.py @@ -1 +1 @@ -__version__ = "0.3.5" +__version__ = "0.3.6" diff --git a/omlx/adapter/gemma4.py b/omlx/adapter/gemma4.py index 17c173332..aa2faee96 100644 --- a/omlx/adapter/gemma4.py +++ b/omlx/adapter/gemma4.py @@ -205,14 +205,15 @@ def extract_gemma4_messages( i += 1 if tool_responses: - processed.append( - { - "role": "assistant", - "content": "", - "tool_responses": tool_responses, - _PRESERVE_BOUNDARY_KEY: True, - } - ) + # Attach tool_responses to the SAME assistant message that + # has tool_calls. The Gemma 4 chat template checks for + # tool_responses on the current message (lines 261-267) + # BEFORE falling back to a forward-scan for role='tool' + # messages (lines 268-302). Putting them on a separate + # assistant message causes both paths to miss, producing a + # corrupt bare <|tool_response> tag and making the model + # loop on the same tool call. + out_msg["tool_responses"] = tool_responses continue # All other roles (user, system) diff --git a/omlx/admin/accuracy_benchmark.py b/omlx/admin/accuracy_benchmark.py index 755fca07e..74eb7fcd2 100644 --- a/omlx/admin/accuracy_benchmark.py +++ b/omlx/admin/accuracy_benchmark.py @@ -33,9 +33,10 @@ _engine_pool_ref: Any = None VALID_BENCHMARKS = [ - "mmlu", "kmmlu", "cmmlu", "jmmlu", + "mmlu", "mmlu_pro", "kmmlu", "cmmlu", "jmmlu", "hellaswag", "truthfulqa", "arc_challenge", "winogrande", - "gsm8k", "humaneval", "mbpp", "livecodebench", + "gsm8k", "mathqa", "humaneval", "mbpp", "livecodebench", + "bbq", "safetybench", ] @@ -440,6 +441,7 @@ async def on_progress(current: int, total: int) -> None: "predicted": qr.predicted, "question": qr.question_text, "raw_response": qr.raw_response, + "category": qr.category, "time_s": round(qr.time_seconds, 3), } for qr in result.question_results diff --git a/omlx/admin/i18n/en.json b/omlx/admin/i18n/en.json index 8b3b79a53..ec91f89e1 100644 --- a/omlx/admin/i18n/en.json +++ b/omlx/admin/i18n/en.json @@ -196,6 +196,8 @@ "models.oq.cancel_tooltip": "Cancel quantization", "models.oq.remove_tooltip": "Remove from list", "models.oq.advanced_settings": "Advanced Settings", + "models.oq.dtype_label": "Non-quant weight dtype", + "models.oq.dtype_help": "float16 gives ~20% faster prefill on M1/M2 Apple Silicon (native fp16). bfloat16 is safer on M3/M4 and for numerical stability. Output name appends '-fp16' when float16 is selected.", "models.uploader.section_label": "Hub Upload", "models.uploader.heading": "Upload oQ Models", "models.uploader.description": "Upload your locally quantized oQ models to HuggingFace Hub.", @@ -428,6 +430,26 @@ "modal.model_settings.cancel": "Cancel", "modal.model_settings.save": "Save", "modal.model_settings.saving": "Saving...", + "modal.model_settings.profiles.section_label": "Profiles", + "modal.model_settings.profiles.templates_row": "Templates (global)", + "modal.model_settings.profiles.profiles_row": "Profiles (this model)", + "modal.model_settings.profiles.new_template": "+ New template", + "modal.model_settings.profiles.new_profile": "+ New profile", + "modal.model_settings.profiles.name_label": "Name", + "modal.model_settings.profiles.name_placeholder": "ID e.g. coding-v1", + "modal.model_settings.profiles.display_name_label": "Display name", + "modal.model_settings.profiles.description_label": "Description (optional)", + "modal.model_settings.profiles.also_as_template": "Also save as global template", + "modal.model_settings.profiles.save": "Save", + "modal.model_settings.profiles.cancel": "Cancel", + "modal.model_settings.profiles.update_from_form": "Update to current form values", + "modal.model_settings.profiles.resync_template": "Re-sync from template", + "modal.model_settings.profiles.delete": "Delete", + "modal.model_settings.profiles.delete_confirm": "Delete this profile?", + "modal.model_settings.profiles.drift_hint": "Form has unsaved changes vs. profile", + "modal.model_settings.profiles.custom_label": "custom", + "modal.model_settings.profiles.invalid_name": "Name must be lowercase letters, digits, underscore, or dash (1-32 chars)", + "models.list.profile_chip": "Profile", "logs.section_label": "Logs", "logs.heading": "Server Logs", "logs.description": "View real-time server logs with auto-refresh.", @@ -589,6 +611,8 @@ "chat.error.invalid_image_type": "Please select an image file", "chat.error.image_too_large": "Image size must be less than 10MB", "chat.error.image_load_failed": "Failed to load image. Please try again.", + "chat.tools_enabled": "Tools enabled", + "chat.tools_disabled": "Tools disabled", "js.error.required_fields": "Required fields cannot be empty: {fields}", "js.error.api_key_min_length": "API key must be at least 4 characters", "js.error.api_key_no_whitespace": "API key must not contain whitespace", diff --git a/omlx/admin/i18n/ja.json b/omlx/admin/i18n/ja.json index d2f10f22c..ead13beea 100644 --- a/omlx/admin/i18n/ja.json +++ b/omlx/admin/i18n/ja.json @@ -196,6 +196,8 @@ "models.oq.cancel_tooltip": "量子化をキャンセル", "models.oq.remove_tooltip": "リストから削除", "models.oq.advanced_settings": "詳細設定", + "models.oq.dtype_label": "Non-quant weight dtype", + "models.oq.dtype_help": "float16 gives ~20% faster prefill on M1/M2 Apple Silicon (native fp16). bfloat16 is safer on M3/M4 and for numerical stability. Output name appends '-fp16' when float16 is selected.", "models.uploader.section_label": "Hub Upload", "models.uploader.heading": "Upload oQ Models", "models.uploader.description": "Upload your locally quantized oQ models to HuggingFace Hub.", @@ -428,6 +430,26 @@ "modal.model_settings.cancel": "キャンセル", "modal.model_settings.save": "保存", "modal.model_settings.saving": "保存中...", + "modal.model_settings.profiles.section_label": "プロファイル", + "modal.model_settings.profiles.templates_row": "テンプレート(グローバル)", + "modal.model_settings.profiles.profiles_row": "プロファイル(このモデル)", + "modal.model_settings.profiles.new_template": "+ 新規テンプレート", + "modal.model_settings.profiles.new_profile": "+ 新規プロファイル", + "modal.model_settings.profiles.name_label": "名前", + "modal.model_settings.profiles.name_placeholder": "ID 例: coding-v1", + "modal.model_settings.profiles.display_name_label": "表示名", + "modal.model_settings.profiles.description_label": "説明(任意)", + "modal.model_settings.profiles.also_as_template": "グローバルテンプレートとしても保存", + "modal.model_settings.profiles.save": "保存", + "modal.model_settings.profiles.cancel": "キャンセル", + "modal.model_settings.profiles.update_from_form": "現在のフォーム値で更新", + "modal.model_settings.profiles.resync_template": "テンプレートから再同期", + "modal.model_settings.profiles.delete": "削除", + "modal.model_settings.profiles.delete_confirm": "このプロファイルを削除しますか?", + "modal.model_settings.profiles.drift_hint": "フォームにプロファイルと異なる未保存の変更があります", + "modal.model_settings.profiles.custom_label": "カスタム", + "modal.model_settings.profiles.invalid_name": "名前は小文字・数字・アンダースコア・ハイフンのみ(1-32 文字)", + "models.list.profile_chip": "プロファイル", "logs.section_label": "ログ", "logs.heading": "サーバーログ", "logs.description": "自動更新でリアルタイムのサーバーログを確認します。", diff --git a/omlx/admin/i18n/ko.json b/omlx/admin/i18n/ko.json index f4326ee5f..26265e415 100644 --- a/omlx/admin/i18n/ko.json +++ b/omlx/admin/i18n/ko.json @@ -196,6 +196,8 @@ "models.oq.cancel_tooltip": "양자화 취소", "models.oq.remove_tooltip": "목록에서 제거", "models.oq.advanced_settings": "고급 설정", + "models.oq.dtype_label": "Non-quant weight dtype", + "models.oq.dtype_help": "float16 gives ~20% faster prefill on M1/M2 Apple Silicon (native fp16). bfloat16 is safer on M3/M4 and for numerical stability. Output name appends '-fp16' when float16 is selected.", "models.uploader.section_label": "Hub Upload", "models.uploader.heading": "Upload oQ Models", "models.uploader.description": "Upload your locally quantized oQ models to HuggingFace Hub.", @@ -428,6 +430,26 @@ "modal.model_settings.cancel": "취소", "modal.model_settings.save": "저장", "modal.model_settings.saving": "저장 중...", + "modal.model_settings.profiles.section_label": "프로필", + "modal.model_settings.profiles.templates_row": "템플릿 (전역)", + "modal.model_settings.profiles.profiles_row": "프로필 (이 모델)", + "modal.model_settings.profiles.new_template": "+ 새 템플릿", + "modal.model_settings.profiles.new_profile": "+ 새 프로필", + "modal.model_settings.profiles.name_label": "이름", + "modal.model_settings.profiles.name_placeholder": "ID 예: coding-v1", + "modal.model_settings.profiles.display_name_label": "표시 이름", + "modal.model_settings.profiles.description_label": "설명 (선택)", + "modal.model_settings.profiles.also_as_template": "전역 템플릿으로도 저장", + "modal.model_settings.profiles.save": "저장", + "modal.model_settings.profiles.cancel": "취소", + "modal.model_settings.profiles.update_from_form": "현재 폼 값으로 업데이트", + "modal.model_settings.profiles.resync_template": "템플릿에서 다시 동기화", + "modal.model_settings.profiles.delete": "삭제", + "modal.model_settings.profiles.delete_confirm": "이 프로필을 삭제하시겠습니까?", + "modal.model_settings.profiles.drift_hint": "폼에 프로필과 다른 저장되지 않은 변경사항이 있습니다", + "modal.model_settings.profiles.custom_label": "사용자 지정", + "modal.model_settings.profiles.invalid_name": "이름은 소문자, 숫자, 밑줄, 하이픈만 허용 (1-32자)", + "models.list.profile_chip": "프로필", "logs.section_label": "로그", "logs.heading": "서버 로그", "logs.description": "실시간 서버 로그를 자동 새로고침으로 확인합니다.", diff --git a/omlx/admin/i18n/zh-TW.json b/omlx/admin/i18n/zh-TW.json index 7911283f3..bc1494091 100644 --- a/omlx/admin/i18n/zh-TW.json +++ b/omlx/admin/i18n/zh-TW.json @@ -196,6 +196,8 @@ "models.oq.cancel_tooltip": "取消量化", "models.oq.remove_tooltip": "從列表中移除", "models.oq.advanced_settings": "進階設定", + "models.oq.dtype_label": "Non-quant weight dtype", + "models.oq.dtype_help": "float16 gives ~20% faster prefill on M1/M2 Apple Silicon (native fp16). bfloat16 is safer on M3/M4 and for numerical stability. Output name appends '-fp16' when float16 is selected.", "models.uploader.section_label": "Hub Upload", "models.uploader.heading": "Upload oQ Models", "models.uploader.description": "Upload your locally quantized oQ models to HuggingFace Hub.", @@ -428,6 +430,26 @@ "modal.model_settings.cancel": "取消", "modal.model_settings.save": "儲存", "modal.model_settings.saving": "儲存中...", + "modal.model_settings.profiles.section_label": "設定檔", + "modal.model_settings.profiles.templates_row": "範本(全域)", + "modal.model_settings.profiles.profiles_row": "設定檔(目前模型)", + "modal.model_settings.profiles.new_template": "+ 新增範本", + "modal.model_settings.profiles.new_profile": "+ 新增設定檔", + "modal.model_settings.profiles.name_label": "名稱", + "modal.model_settings.profiles.name_placeholder": "ID,例如 coding-v1", + "modal.model_settings.profiles.display_name_label": "顯示名稱", + "modal.model_settings.profiles.description_label": "說明(選填)", + "modal.model_settings.profiles.also_as_template": "同時儲存為全域範本", + "modal.model_settings.profiles.save": "儲存", + "modal.model_settings.profiles.cancel": "取消", + "modal.model_settings.profiles.update_from_form": "更新為目前表單值", + "modal.model_settings.profiles.resync_template": "從範本重新同步", + "modal.model_settings.profiles.delete": "刪除", + "modal.model_settings.profiles.delete_confirm": "刪除此設定檔?", + "modal.model_settings.profiles.drift_hint": "表單相對設定檔有未儲存的變動", + "modal.model_settings.profiles.custom_label": "自訂", + "modal.model_settings.profiles.invalid_name": "名稱僅允許小寫字母、數字、底線、連字號(1-32 字元)", + "models.list.profile_chip": "設定檔", "logs.section_label": "日誌", "logs.heading": "伺服器 Logs", "logs.description": "檢視即時伺服器 Logs,支援自動重新整理。", diff --git a/omlx/admin/i18n/zh.json b/omlx/admin/i18n/zh.json index 66356782e..748cb0956 100644 --- a/omlx/admin/i18n/zh.json +++ b/omlx/admin/i18n/zh.json @@ -196,6 +196,8 @@ "models.oq.cancel_tooltip": "取消量化", "models.oq.remove_tooltip": "从列表中移除", "models.oq.advanced_settings": "高级设置", + "models.oq.dtype_label": "Non-quant weight dtype", + "models.oq.dtype_help": "float16 gives ~20% faster prefill on M1/M2 Apple Silicon (native fp16). bfloat16 is safer on M3/M4 and for numerical stability. Output name appends '-fp16' when float16 is selected.", "models.uploader.section_label": "Hub Upload", "models.uploader.heading": "Upload oQ Models", "models.uploader.description": "Upload your locally quantized oQ models to HuggingFace Hub.", @@ -428,6 +430,26 @@ "modal.model_settings.cancel": "取消", "modal.model_settings.save": "保存", "modal.model_settings.saving": "保存中...", + "modal.model_settings.profiles.section_label": "配置档", + "modal.model_settings.profiles.templates_row": "模板(全局)", + "modal.model_settings.profiles.profiles_row": "配置档(当前模型)", + "modal.model_settings.profiles.new_template": "+ 新建模板", + "modal.model_settings.profiles.new_profile": "+ 新建配置档", + "modal.model_settings.profiles.name_label": "名称", + "modal.model_settings.profiles.name_placeholder": "ID,例如 coding-v1", + "modal.model_settings.profiles.display_name_label": "显示名", + "modal.model_settings.profiles.description_label": "描述(可选)", + "modal.model_settings.profiles.also_as_template": "同时保存为全局模板", + "modal.model_settings.profiles.save": "保存", + "modal.model_settings.profiles.cancel": "取消", + "modal.model_settings.profiles.update_from_form": "更新为当前表单值", + "modal.model_settings.profiles.resync_template": "从模板重新同步", + "modal.model_settings.profiles.delete": "删除", + "modal.model_settings.profiles.delete_confirm": "删除此配置档?", + "modal.model_settings.profiles.drift_hint": "表单相对配置档有未保存的改动", + "modal.model_settings.profiles.custom_label": "自定义", + "modal.model_settings.profiles.invalid_name": "名称仅允许小写字母、数字、下划线、短横线(1-32 字符)", + "models.list.profile_chip": "配置档", "logs.section_label": "日志", "logs.heading": "服务器日志", "logs.description": "查看实时服务器日志,支持自动刷新。", diff --git a/omlx/admin/oq_manager.py b/omlx/admin/oq_manager.py index 355e02573..fcf6ebfc9 100644 --- a/omlx/admin/oq_manager.py +++ b/omlx/admin/oq_manager.py @@ -73,6 +73,7 @@ class QuantTask: group_size: int = 64 sensitivity_model_path: str = "" text_only: bool = False + dtype: str = "bfloat16" def to_dict(self) -> dict: """Serialize task to JSON-compatible dict.""" @@ -92,6 +93,7 @@ def to_dict(self) -> dict: "completed_at": self.completed_at, "source_size": self.source_size, "output_size": self.output_size, + "dtype": self.dtype, } @@ -219,12 +221,15 @@ async def start_quantization( group_size: int = 64, sensitivity_model_path: str = "", text_only: bool = False, + dtype: str = "bfloat16", ) -> QuantTask: """Start a quantization job. Args: model_path: Path to source model directory. oq_level: oQ level (2, 3, 4, 6, or 8). + dtype: Target fp dtype for non-quantized weights and quant + scales/biases. "bfloat16" (default) or "float16". Returns: The created QuantTask. @@ -232,19 +237,23 @@ async def start_quantization( Raises: ValueError: On invalid inputs or output conflict. """ - from ..oq import OQ_LEVELS, resolve_output_name + from ..oq import OQ_DTYPES, OQ_LEVELS, resolve_output_name if oq_level not in OQ_LEVELS: raise ValueError( f"Invalid oQ level {oq_level}. Must be one of {sorted(OQ_LEVELS)}" ) + if dtype not in OQ_DTYPES: + raise ValueError( + f"Invalid dtype {dtype!r}. Must be one of {OQ_DTYPES}" + ) source = Path(model_path) if not source.exists() or not (source / "config.json").exists(): raise ValueError(f"Model not found: {model_path}") model_name = source.name - output_name = resolve_output_name(model_name, oq_level) + output_name = resolve_output_name(model_name, oq_level, dtype) output_path = self._output_dir / output_name if output_path.exists(): @@ -253,16 +262,17 @@ async def start_quantization( "Delete it first via the Manager tab." ) - # Check for duplicate active tasks + # Check for duplicate active tasks (same level + dtype combo) for task in self._tasks.values(): if ( task.model_path == model_path and task.oq_level == oq_level + and task.dtype == dtype and task.status in _ACTIVE_STATUSES ): raise ValueError( f"Quantization for '{model_name}' at oQ{oq_level:g} " - "is already in progress" + f"({dtype}) is already in progress" ) source_size = sum( @@ -283,6 +293,7 @@ async def start_quantization( group_size=group_size, sensitivity_model_path=sensitivity_model_path, text_only=text_only, + dtype=dtype, ) self._tasks[task_id] = task @@ -438,6 +449,7 @@ def _progress_cb(phase: str, pct: float) -> None: None, # target_bpw None, # hard_cap_bpw task.sensitivity_model_path, + task.dtype, ) if task_id in self._cancelled: diff --git a/omlx/admin/routes.py b/omlx/admin/routes.py index e697c3aa0..374a7b005 100644 --- a/omlx/admin/routes.py +++ b/omlx/admin/routes.py @@ -25,7 +25,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse from fastapi.templating import Jinja2Templates -from pydantic import BaseModel +from pydantic import BaseModel, Field from .auth import ( REMEMBER_ME_MAX_AGE, @@ -127,6 +127,42 @@ class ModelSettingsRequest(BaseModel): is_default: Optional[bool] = None +class CreateProfileRequest(BaseModel): + """Request body for creating a per-model profile.""" + name: str + display_name: str + description: Optional[str] = None + settings: Dict[str, Any] = Field(default_factory=dict) + also_save_as_template: bool = False + source_template: Optional[str] = None + + +class UpdateProfileRequest(BaseModel): + """Request body for updating/renaming a per-model profile.""" + new_name: Optional[str] = None + display_name: Optional[str] = None + description: Optional[str] = None + settings: Optional[Dict[str, Any]] = None + source_template: Optional[str] = None + also_save_as_template: bool = False + + +class CreateTemplateRequest(BaseModel): + """Request body for creating a global template.""" + name: str + display_name: str + description: Optional[str] = None + settings: Dict[str, Any] = Field(default_factory=dict) + + +class UpdateTemplateRequest(BaseModel): + """Request body for updating/renaming a global template.""" + new_name: Optional[str] = None + display_name: Optional[str] = None + description: Optional[str] = None + settings: Optional[Dict[str, Any]] = None + + class GlobalSettingsRequest(BaseModel): """Request model for updating global server settings.""" @@ -236,6 +272,7 @@ class OQStartRequest(BaseModel): group_size: int = 64 sensitivity_model_path: str = "" text_only: bool = False + dtype: str = "bfloat16" class HFUploadRequest(BaseModel): @@ -1347,6 +1384,7 @@ async def list_models(is_admin: bool = Depends(require_admin)): "model_type": model_info.get("model_type", "llm"), "config_model_type": model_info.get("config_model_type", ""), "thinking_default": model_info.get("thinking_default"), + "preserve_thinking_default": model_info.get("preserve_thinking_default"), "last_access": model_info.get("last_access"), } @@ -1386,6 +1424,7 @@ async def list_models(is_admin: bool = Depends(require_admin)): "is_default": settings.is_default, "display_name": settings.display_name, "description": settings.description, + "active_profile_name": settings.active_profile_name, } models.append(model_data) @@ -1621,6 +1660,22 @@ async def update_model_settings( if request.is_default and server_state: server_state.default_model = model_id + # If an active profile was set, clear it when the user's save diverges + # from the profile's stored values. + if current_settings.active_profile_name: + profile = settings_manager.get_profile( + model_id, current_settings.active_profile_name + ) + if profile is None: + current_settings.active_profile_name = None + else: + profile_settings = profile.get("settings", {}) or {} + candidate = current_settings.to_dict() + for key, expected in profile_settings.items(): + if candidate.get(key) != expected: + current_settings.active_profile_name = None + break + # Persist settings settings_manager.set_settings(model_id, current_settings) @@ -1650,6 +1705,221 @@ async def update_model_settings( } +# ============================================================================= +# Profile & Template endpoints +# ============================================================================= + + +def _require_settings_manager(): + mgr = _get_settings_manager() + if mgr is None: + raise HTTPException(status_code=503, detail="Server not initialized") + return mgr + + +def _require_model(model_id: str): + pool = _get_engine_pool() + if pool is None: + raise HTTPException(status_code=503, detail="Engine pool not initialized") + entry = pool.get_entry(model_id) + if entry is None: + raise HTTPException(status_code=404, detail=f"Model not found: {model_id}") + return entry + + +@router.get("/api/models/{model_id}/profiles") +async def list_model_profiles( + model_id: str, + is_admin: bool = Depends(require_admin), +): + mgr = _require_settings_manager() + _require_model(model_id) + return {"profiles": mgr.list_profiles(model_id)} + + +@router.post("/api/models/{model_id}/profiles") +async def create_model_profile( + model_id: str, + request: CreateProfileRequest, + is_admin: bool = Depends(require_admin), +): + from ..model_profiles import InvalidProfileNameError, filter_universal_fields + + mgr = _require_settings_manager() + _require_model(model_id) + try: + profile = mgr.save_profile( + model_id=model_id, + name=request.name, + display_name=request.display_name, + description=request.description, + settings=request.settings or {}, + source_template=request.source_template, + ) + except InvalidProfileNameError as e: + raise HTTPException(status_code=400, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=409, detail=str(e)) + + if request.also_save_as_template: + try: + mgr.upsert_template( + name=request.name, + display_name=request.display_name, + description=request.description, + settings=filter_universal_fields(request.settings or {}), + ) + except InvalidProfileNameError as e: + raise HTTPException(status_code=400, detail=str(e)) + return {"profile": profile} + + +@router.put("/api/models/{model_id}/profiles/{name}") +async def update_model_profile( + model_id: str, + name: str, + request: UpdateProfileRequest, + is_admin: bool = Depends(require_admin), +): + from ..model_profiles import InvalidProfileNameError, filter_universal_fields + + mgr = _require_settings_manager() + _require_model(model_id) + try: + updated = mgr.update_profile( + model_id=model_id, + name=name, + new_name=request.new_name, + display_name=request.display_name, + description=request.description, + settings=request.settings, + source_template=request.source_template, + ) + except InvalidProfileNameError as e: + raise HTTPException(status_code=400, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=409, detail=str(e)) + if updated is None: + raise HTTPException(status_code=404, detail=f"Profile not found: {name}") + + if request.also_save_as_template and request.settings is not None: + try: + mgr.upsert_template( + name=updated["name"], + display_name=updated["display_name"], + description=updated.get("description"), + settings=filter_universal_fields(request.settings), + ) + except InvalidProfileNameError as e: + raise HTTPException(status_code=400, detail=str(e)) + return {"profile": updated} + + +@router.delete("/api/models/{model_id}/profiles/{name}") +async def delete_model_profile( + model_id: str, + name: str, + is_admin: bool = Depends(require_admin), +): + mgr = _require_settings_manager() + _require_model(model_id) + if not mgr.delete_profile(model_id, name): + raise HTTPException(status_code=404, detail=f"Profile not found: {name}") + return {"deleted": True, "name": name} + + +@router.post("/api/models/{model_id}/profiles/{name}/apply") +async def apply_model_profile( + model_id: str, + name: str, + is_admin: bool = Depends(require_admin), +): + mgr = _require_settings_manager() + _require_model(model_id) + applied = mgr.apply_profile(model_id, name) + if applied is None: + raise HTTPException(status_code=404, detail=f"Profile not found: {name}") + return {"model_id": model_id, "settings": applied.to_dict()} + + +@router.get("/api/profile-fields") +async def get_profile_fields(is_admin: bool = Depends(require_admin)): + from ..model_profiles import ( + UNIVERSAL_PROFILE_FIELDS, + MODEL_SPECIFIC_PROFILE_FIELDS, + ) + + return { + "universal": list(UNIVERSAL_PROFILE_FIELDS), + "model_specific": list(MODEL_SPECIFIC_PROFILE_FIELDS), + } + + +@router.get("/api/profile-templates") +async def list_templates(is_admin: bool = Depends(require_admin)): + mgr = _require_settings_manager() + return {"templates": mgr.list_templates()} + + +@router.post("/api/profile-templates") +async def create_template( + request: CreateTemplateRequest, + is_admin: bool = Depends(require_admin), +): + from ..model_profiles import InvalidProfileNameError + + mgr = _require_settings_manager() + try: + tmpl = mgr.save_template( + name=request.name, + display_name=request.display_name, + description=request.description, + settings=request.settings or {}, + ) + except InvalidProfileNameError as e: + raise HTTPException(status_code=400, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=409, detail=str(e)) + return {"template": tmpl} + + +@router.put("/api/profile-templates/{name}") +async def update_template( + name: str, + request: UpdateTemplateRequest, + is_admin: bool = Depends(require_admin), +): + from ..model_profiles import InvalidProfileNameError + + mgr = _require_settings_manager() + try: + updated = mgr.update_template( + name=name, + new_name=request.new_name, + display_name=request.display_name, + description=request.description, + settings=request.settings, + ) + except InvalidProfileNameError as e: + raise HTTPException(status_code=400, detail=str(e)) + except ValueError as e: + raise HTTPException(status_code=409, detail=str(e)) + if updated is None: + raise HTTPException(status_code=404, detail=f"Template not found: {name}") + return {"template": updated} + + +@router.delete("/api/profile-templates/{name}") +async def delete_template( + name: str, + is_admin: bool = Depends(require_admin), +): + mgr = _require_settings_manager() + if not mgr.delete_template(name): + raise HTTPException(status_code=404, detail=f"Template not found: {name}") + return {"deleted": True, "name": name} + + @router.get("/api/models/{model_id}/generation_config") async def get_generation_config( model_id: str, @@ -4130,6 +4400,11 @@ async def start_oq_quantization( status_code=400, detail="Invalid oQ level. Must be 2, 3, 4, 5, 6, or 8", ) + if request.dtype not in ("bfloat16", "float16"): + raise HTTPException( + status_code=400, + detail="Invalid dtype. Must be 'bfloat16' or 'float16'", + ) try: task = await _oq_manager.start_quantization( model_path=request.model_path, @@ -4137,6 +4412,7 @@ async def start_oq_quantization( group_size=request.group_size, sensitivity_model_path=request.sensitivity_model_path, text_only=request.text_only, + dtype=request.dtype, ) return {"success": True, "task": task.to_dict()} except ValueError as e: @@ -4288,3 +4564,389 @@ async def remove_upload_task( status_code=404, detail="Task not found or still active" ) return {"success": True} + + +# ============================================================================= +# MCP Server Management Routes +# ============================================================================= + +# In-memory store for pending OAuth PKCE sessions, keyed by state parameter +_mcp_oauth_sessions: Dict[str, Dict] = {} + + +def _get_mcp_config_path() -> Optional[Path]: + """Return the MCP config file path currently in use (or None).""" + from ..mcp.config import _find_config_file + try: + return _find_config_file() + except Exception: + return None + + +def _read_mcp_config_file() -> Optional[Dict]: + """Read and parse the MCP config JSON file.""" + path = _get_mcp_config_path() + if path is None: + return None + try: + return json.loads(Path(path).read_text()) + except Exception: + return None + + +def _write_mcp_config_file(config_data: Dict) -> Path: + """Write config_data to the MCP config file, creating it if needed.""" + path = _get_mcp_config_path() + if path is None: + path = Path("./mcp.json") + Path(path).write_text(json.dumps(config_data, indent=2)) + return path + + +def _mcp_oauth_result_html(success: bool, message: str) -> str: + icon = "✓" if success else "✗" + color = "#16a34a" if success else "#dc2626" + bg = "#f0fdf4" if success else "#fef2f2" + js_success = "true" if success else "false" + return f""" + +
+{message}
+