Merge pull request #83 from askui/feat/introduce-claude-4

feat!: introduce Claude 4 Sonnet

Author: adi-wan-askui

README.md

@@ -128,8 +128,8 @@ with VisionAgent() as agent:
 
     agent.click("search field")
 
-    # Use Anthropic (Claude 3.5 Sonnet V2) as model
-    agent.click("search field", model="anthropic-claude-3-5-sonnet-20241022")
+    # Use Anthropic (Claude 4 Sonnet) as model
+    agent.click("search field", model="claude-sonnet-4-20250514")
 ```
 
 
@@ -201,7 +201,7 @@ with VisionAgent(model="askui-combo") as agent:
 
 # Use different models for different tasks
 with VisionAgent(model={
-    "act": "anthropic-claude-3-5-sonnet-20241022",  # Use Claude for act()
+    "act": "claude-sonnet-4-20250514",  # Use Claude for act()
     "get": "askui",  # Use AskUI for get()
     "locate": "askui-combo",  # Use AskUI combo for locate() (and click(), mouse_move())
 }) as agent:
@@ -240,7 +240,7 @@ Supported commands are: `act()`, `click()`, `get()`, `locate()`, `mouse_move()`
 Supported commands are: `act()`, `get()`, `click()`, `locate()`, `mouse_move()`
 | Model Name  | Info | Execution Speed | Security | Cost | Reliability |
 |-------------|--------------------|--------------|--------------|--------------|--------------|
-| `anthropic-claude-3-5-sonnet-20241022` | The [Computer Use](https://docs.anthropic.com/en/docs/agents-and-tools/computer-use) model from Antrophic is a Large Action Model (LAM), which can autonomously achieve goals. e.g. `"Book me a flight from Berlin to Rom"` | slow, >1s per step | Model hosting by Anthropic | High, up to 1,5$ per act | Not recommended for production usage |
+| `claude-sonnet-4-20250514` | The [Computer Use](https://docs.anthropic.com/en/docs/agents-and-tools/computer-use) model from Antrophic is a Large Action Model (LAM), which can autonomously achieve goals. e.g. `"Book me a flight from Berlin to Rom"` | slow, >1s per step | Model hosting by Anthropic | High, up to 1,5$ per act | Not recommended for production usage |
 > **Note:** Configure your Antrophic Model Provider [here](#3a-authenticate-with-an-ai-model-provider)
 
 
@@ -409,7 +409,7 @@ custom_models: ModelRegistry = {
     "dynamic-model": lambda: create_custom_model("your-api-key"),
     "dynamic-model-cached": lambda: create_custom_model_cached("your-api-key"),
     "askui": lambda: create_custom_model_cached("your-api-key"), # overrides default model
-    "anthropic-claude-3-5-sonnet-20241022": lambda: create_custom_model_cached("your-api-key"), # overrides model
+    "claude-sonnet-4-20250514": lambda: create_custom_model_cached("your-api-key"), # overrides model
 }
 
 
@@ -735,7 +735,7 @@ with VisionAgent() as agent:
 
 **AskUI Vision Agent** is a versatile AI powered framework that enables you to automate computer tasks in Python.
 
-It connects Agent OS with powerful computer use models like Anthropic's Claude Sonnet 3.5 v2 and the AskUI Prompt-to-Action series. It is your entry point for building complex automation scenarios with detailed instructions or let the agent explore new challenges on its own.
+It connects Agent OS with powerful computer use models like Anthropic's Claude Sonnet 4 and the AskUI Prompt-to-Action series. It is your entry point for building complex automation scenarios with detailed instructions or let the agent explore new challenges on its own.
 
 
 ![image](docs/assets/Architecture.svg)

src/askui/agent.py

@@ -11,7 +11,7 @@
 from askui.models.shared.computer_agent_cb_param import OnMessageCb
 from askui.models.shared.computer_agent_message_param import MessageParam
 from askui.models.shared.tools import ToolCollection
-from askui.tools.computer import Computer20241022Tool
+from askui.tools.computer import Computer20241022Tool, Computer20250124Tool
 from askui.tools.exception_tool import ExceptionTool
 from askui.utils.image_utils import ImageSource, Img
 
@@ -81,10 +81,13 @@ def __init__(
                 reporter=self._reporter,
             ),
         )
+        self._tool_collection = ToolCollection(
+            tools=[
+                ExceptionTool(),
+            ]
+        )
         _models = initialize_default_model_registry(
-            tool_collection=ToolCollection(
-                tools=[Computer20241022Tool(self.tools.os), ExceptionTool()]
-            ),
+            tool_collection=self._tool_collection,
             reporter=self._reporter,
         )
         _models.update(models or {})
@@ -628,6 +631,11 @@ def act(
         Returns:
             None
 
+        Raises:
+            MaxTokensExceededError: If the model reaches the maximum token limit
+                defined in the agent settings.
+            ModelRefusalError: If the model refuses to process the request.
+
         Example:
             ```python
             from askui import VisionAgent
@@ -650,7 +658,19 @@ def act(
         messages: list[MessageParam] = (
             [MessageParam(role="user", content=goal)] if isinstance(goal, str) else goal
         )
-        self._model_router.act(messages, model or self._model_choice["act"], on_message)
+        _model = model or self._model_choice["act"]
+        self._update_tool_collection(_model)
+        self._model_router.act(messages, _model, on_message)
+
+    def _update_tool_collection(self, model: str) -> None:
+        if model == ModelName.ANTHROPIC__CLAUDE__3_5__SONNET__20241022:
+            self._tool_collection.append_tool(
+                Computer20241022Tool(agent_os=self.tools.os)
+            )
+        if model == ModelName.CLAUDE__SONNET__4__20250514 or model == ModelName.ASKUI:
+            self._tool_collection.append_tool(
+                Computer20250124Tool(agent_os=self.tools.os)
+            )
 
     @telemetry.record_call()
     @validate_call

src/askui/models/anthropic/computer_agent.py

@@ -1,17 +1,22 @@
 from typing import TYPE_CHECKING, cast
 
-from anthropic import Anthropic
+from anthropic import NOT_GIVEN, Anthropic, NotGiven
+from anthropic.types import AnthropicBetaParam
 from typing_extensions import override
 
 from askui.models.anthropic.settings import ClaudeComputerAgentSettings
 from askui.models.models import ANTHROPIC_MODEL_NAME_MAPPING, ModelName
-from askui.models.shared.computer_agent import ComputerAgent
+from askui.models.shared.computer_agent import (
+    COMPUTER_USE_20241022_BETA_FLAG,
+    COMPUTER_USE_20250124_BETA_FLAG,
+    ComputerAgent,
+)
 from askui.models.shared.computer_agent_message_param import MessageParam
 from askui.models.shared.tools import ToolCollection
 from askui.reporting import Reporter
 
 if TYPE_CHECKING:
-    from anthropic.types.beta import BetaMessageParam
+    from anthropic.types.beta import BetaMessageParam, BetaThinkingConfigParam
 
 
 class ClaudeComputerAgent(ComputerAgent[ClaudeComputerAgentSettings]):
@@ -26,20 +31,31 @@ def __init__(
             api_key=self._settings.anthropic.api_key.get_secret_value()
         )
 
+    def _get_betas(self, model_choice: str) -> list[AnthropicBetaParam] | NotGiven:
+        if model_choice == ModelName.ANTHROPIC__CLAUDE__3_5__SONNET__20241022:
+            return self._settings.betas + [COMPUTER_USE_20241022_BETA_FLAG]
+        if model_choice == ModelName.CLAUDE__SONNET__4__20250514:
+            return self._settings.betas + [COMPUTER_USE_20250124_BETA_FLAG]
+        return NOT_GIVEN
+
     @override
     def _create_message(
         self, messages: list[MessageParam], model_choice: str
     ) -> MessageParam:
         response = self._client.beta.messages.with_raw_response.create(
             max_tokens=self._settings.max_tokens,
             messages=[
-                cast("BetaMessageParam", message.model_dump(mode="json"))
+                cast("BetaMessageParam", message.model_dump(exclude={"stop_reason"}))
                 for message in messages
             ],
             model=ANTHROPIC_MODEL_NAME_MAPPING[ModelName(model_choice)],
             system=[self._system],
             tools=self._tool_collection.to_params(),
-            betas=self._settings.betas,
+            betas=self._get_betas(model_choice),
+            thinking=cast(
+                "BetaThinkingConfigParam", self._settings.thinking.model_dump()
+            ),
+            tool_choice=self._settings.tool_choice,
         )
         parsed_response = response.parse()
         return MessageParam.model_validate(parsed_response.model_dump())

src/askui/models/anthropic/settings.py

@@ -1,11 +1,9 @@
-from pydantic import BaseModel, Field, SecretStr
+from pydantic import Field, SecretStr
 from pydantic_settings import BaseSettings
 
 from askui.models.shared.computer_agent import ComputerAgentSettingsBase
 from askui.models.shared.settings import ChatCompletionsCreateSettings
 
-COMPUTER_USE_BETA_FLAG = "computer-use-2024-10-22"
-
 
 class AnthropicSettings(BaseSettings):
     api_key: SecretStr = Field(
@@ -15,7 +13,7 @@ class AnthropicSettings(BaseSettings):
     )
 
 
-class ClaudeSettingsBase(BaseModel):
+class ClaudeSettingsBase(BaseSettings):
     anthropic: AnthropicSettings = Field(default_factory=lambda: AnthropicSettings())
 
 

src/askui/models/askui/computer_agent.py

@@ -1,16 +1,34 @@
 import httpx
+from anthropic.types.beta import (
+    BetaTextBlockParam,
+    BetaToolChoiceParam,
+    BetaToolUnionParam,
+)
+from pydantic import BaseModel, ConfigDict
 from tenacity import retry, retry_if_exception, stop_after_attempt, wait_exponential
 from typing_extensions import override
 
 from askui.models.askui.settings import AskUiComputerAgentSettings
-from askui.models.shared.computer_agent import ComputerAgent
+from askui.models.shared.computer_agent import ComputerAgent, ThinkingConfigParam
 from askui.models.shared.computer_agent_message_param import MessageParam
 from askui.models.shared.tools import ToolCollection
 from askui.reporting import Reporter
 
 from ...logger import logger
 
 
+class RequestBody(BaseModel):
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+    max_tokens: int
+    messages: list[MessageParam]
+    model: str
+    tools: list[BetaToolUnionParam]
+    betas: list[str]
+    system: list[BetaTextBlockParam]
+    thinking: ThinkingConfigParam
+    tool_choice: BetaToolChoiceParam
+
+
 def is_retryable_error(exception: BaseException) -> bool:
     """Check if the exception is a retryable error (status codes 429 or 529)."""
     if isinstance(exception, httpx.HTTPStatusError):
@@ -47,21 +65,31 @@ def _create_message(
         model_choice: str,  # noqa: ARG002
     ) -> MessageParam:
         try:
-            request_body = {
-                "max_tokens": self._settings.max_tokens,
-                "messages": [msg.model_dump(mode="json") for msg in messages],
-                "model": self._settings.model,
-                "tools": self._tool_collection.to_params(),
-                "betas": self._settings.betas,
-                "system": [self._system],
-            }
+            request_body = RequestBody(
+                max_tokens=self._settings.max_tokens,
+                messages=messages,
+                model=self._settings.model,
+                tools=self._tool_collection.to_params(),
+                betas=self._settings.betas,
+                system=[self._system],
+                tool_choice=self._settings.tool_choice,
+                thinking=self._settings.thinking,
+            )
             response = self._client.post(
-                "/act/inference", json=request_body, timeout=300.0
+                "/act/inference",
+                json=request_body.model_dump(
+                    mode="json", exclude={"messages": {"stop_reason"}}
+                ),
+                timeout=300.0,
             )
             response.raise_for_status()
-            response_data = response.json()
-            return MessageParam.model_validate(response_data)
+            return MessageParam.model_validate_json(response.text)
         except Exception as e:  # noqa: BLE001
             if is_retryable_error(e):
                 logger.debug(e)
+            if (
+                isinstance(e, httpx.HTTPStatusError)
+                and 400 <= e.response.status_code < 500
+            ):
+                raise ValueError(e.response.json()) from e
             raise

src/askui/models/askui/settings.py

@@ -6,7 +6,12 @@
 
 from askui.models.models import ModelName
 from askui.models.shared.base_agent import AgentSettingsBase
-from askui.models.shared.computer_agent import ComputerAgentSettingsBase
+from askui.models.shared.computer_agent import (
+    COMPUTER_USE_20250124_BETA_FLAG,
+    ComputerAgentSettingsBase,
+    ThinkingConfigEnabledParam,
+    ThinkingConfigParam,
+)
 
 
 class AskUiSettings(BaseSettings):
@@ -40,12 +45,16 @@ def base_url(self) -> str:
 
 
 class AskUiComputerAgentSettings(ComputerAgentSettingsBase):
-    model: str = ModelName.ANTHROPIC__CLAUDE__3_5__SONNET__20241022
+    model: str = ModelName.CLAUDE__SONNET__4__20250514
     askui: AskUiSettings = Field(default_factory=AskUiSettings)
+    betas: list[str] = Field(default_factory=lambda: [COMPUTER_USE_20250124_BETA_FLAG])
+    thinking: ThinkingConfigParam = Field(
+        default_factory=lambda: ThinkingConfigEnabledParam(budget_tokens=2048)
+    )
 
 
 class AskUiAndroidAgentSettings(AgentSettingsBase):
     """Settings for AskUI Android agent."""
 
-    model: str = ModelName.ANTHROPIC__CLAUDE__3_5__SONNET__20241022
+    model: str = ModelName.CLAUDE__SONNET__4__20250514
     askui: AskUiSettings = Field(default_factory=AskUiSettings)

src/askui/models/exceptions.py

@@ -99,3 +99,36 @@ def __init__(
             message=f'Model "{model_choice}" is an instance of {actual_type.mro()}, '
             f"expected it to be an instance of {expected_type.mro()}",
         )
+
+
+class MaxTokensExceededError(AutomationError):
+    """Exception raised when the model stops due to reaching the maximum token limit.
+
+    Args:
+        max_tokens (int): The maximum token limit that was exceeded.
+        message (str, optional): Custom error message. If not provided, a default
+            message will be generated.
+    """
+
+    def __init__(self, max_tokens: int, message: str | None = None):
+        self.max_tokens = max_tokens
+        error_msg = (
+            f"Model stopped due to reaching maximum token limit of {max_tokens} tokens"
+            if message is None
+            else message
+        )
+        super().__init__(error_msg)
+
+
+class ModelRefusalError(AutomationError):
+    """Exception raised when the model refuses to process the request.
+
+    Args:
+        message (str, optional): Custom error message. If not provided, a default
+            message will be generated.
+    """
+
+    def __init__(self, message: str | None = None):
+        super().__init__(
+            "Model refused to process the request" if message is None else message
+        )

src/askui/models/model_router.py

@@ -5,11 +5,7 @@
 
 from askui.locators.locators import Locator
 from askui.locators.serializers import AskUiLocatorSerializer, VlmLocatorSerializer
-from askui.models.anthropic.settings import (
-    AnthropicSettings,
-    ClaudeComputerAgentSettings,
-    ClaudeSettings,
-)
+from askui.models.anthropic.settings import ClaudeComputerAgentSettings, ClaudeSettings
 from askui.models.askui.ai_element_utils import AiElementCollection
 from askui.models.askui.android_agent import AskUiAndroidAgent
 from askui.models.askui.computer_agent import AskUiComputerAgent
@@ -75,18 +71,13 @@ def vlm_locator_serializer() -> VlmLocatorSerializer:
 
     @functools.cache
     def anthropic_facade() -> ModelFacade:
-        settings = AnthropicSettings()
         computer_agent = ClaudeComputerAgent(
             tool_collection=tool_collection,
             reporter=reporter,
-            settings=ClaudeComputerAgentSettings(
-                anthropic=settings,
-            ),
+            settings=ClaudeComputerAgentSettings(),
         )
         handler = ClaudeHandler(
-            settings=ClaudeSettings(
-                anthropic=settings,
-            ),
+            settings=ClaudeSettings(),
             locator_serializer=vlm_locator_serializer(),
         )
         return ModelFacade(
@@ -117,12 +108,13 @@ def hf_spaces_handler() -> HFSpacesHandler:
         )
 
     return {
+        ModelName.ANTHROPIC__CLAUDE__3_5__SONNET__20241022: anthropic_facade,
         ModelName.ASKUI: askui_facade,
         ModelName.ASKUI__AI_ELEMENT: askui_model_router,
         ModelName.ASKUI__COMBO: askui_model_router,
         ModelName.ASKUI__OCR: askui_model_router,
         ModelName.ASKUI__PTA: askui_model_router,
-        ModelName.ANTHROPIC__CLAUDE__3_5__SONNET__20241022: anthropic_facade,
+        ModelName.CLAUDE__SONNET__4__20250514: anthropic_facade,
         ModelName.HF__SPACES__ASKUI__PTA_1: hf_spaces_handler,
         ModelName.HF__SPACES__QWEN__QWEN2_VL_2B_INSTRUCT: hf_spaces_handler,
         ModelName.HF__SPACES__QWEN__QWEN2_VL_7B_INSTRUCT: hf_spaces_handler,