Skip to content

Commit cd4d4e8

Browse files
refactor: update action method return type to include Coordinate
- Modified the return type of the `action` method in `Computer20250124Tool` to include `Coordinate` in addition to `Image.Image | None`, enhancing its functionality to return more comprehensive results.
1 parent a222370 commit cd4d4e8

File tree

2 files changed

+20
-17
lines changed

2 files changed

+20
-17
lines changed

src/askui/tools/computer.py

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,20 @@
44
from dataclasses import dataclass
55
from typing import Annotated, Literal, TypedDict, cast, get_args
66

7-
from anthropic.types.beta import (BetaToolComputerUse20241022Param,
8-
BetaToolComputerUse20250124Param)
7+
from anthropic.types.beta import (
8+
BetaToolComputerUse20241022Param,
9+
BetaToolComputerUse20250124Param,
10+
)
911
from PIL import Image
1012
from pydantic import Field, validate_call
1113
from typing_extensions import Self, override
1214

1315
from askui.tools.agent_os import AgentOs, Coordinate, ModifierKey, PcKey
14-
from askui.utils.image_utils import (scale_coordinates_back,
15-
scale_coordinates_with_padding,
16-
scale_image_with_padding)
16+
from askui.utils.image_utils import (
17+
scale_coordinates_back,
18+
scale_coordinates_with_padding,
19+
scale_image_with_padding,
20+
)
1721

1822
from ..models.shared.tools import InputSchema, Tool
1923

@@ -325,11 +329,17 @@ def _screenshot(self) -> Image.Image:
325329
self._real_screen_height = screenshot.height
326330
return scale_image_with_padding(screenshot, self._width, self._height)
327331

328-
329332
def _get_mouse_position_scaled(self) -> Coordinate:
330333
mouse_position: Coordinate = self._agent_os.get_mouse_position()
331334
real_screen_width, real_screen_height = self._get_real_screen_resolution()
332-
x, y = scale_coordinates_with_padding(mouse_position.x, mouse_position.y, real_screen_width, real_screen_height, self._width, self._height)
335+
x, y = scale_coordinates_with_padding(
336+
mouse_position.x,
337+
mouse_position.y,
338+
real_screen_width,
339+
real_screen_height,
340+
self._width,
341+
self._height,
342+
)
333343
return Coordinate(x=int(x), y=int(y))
334344

335345

@@ -433,7 +443,7 @@ def __call__( # noqa: C901
433443
scroll_amount: Annotated[int, Field(ge=0)] | None = None,
434444
duration: Annotated[float, Field(ge=0.0, le=100.0)] | None = None,
435445
key: str | None = None, # maybe not all keys supported
436-
) -> Image.Image | None:
446+
) -> Image.Image | None | Coordinate:
437447
match action:
438448
case "hold_key":
439449
self._hold_key(keystroke=text, duration=duration) # type: ignore[arg-type]

src/askui/utils/image_utils.py

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@
77
from pathlib import Path
88
from typing import Any, Literal, Tuple, Union
99

10-
from PIL import Image
10+
from PIL import Image, ImageDraw, ImageOps, UnidentifiedImageError
1111
from PIL import Image as PILImage
12-
from PIL import ImageDraw, ImageOps, UnidentifiedImageError
1312
from pydantic import ConfigDict, RootModel, field_validator
1413

1514
# Regex to capture any kind of valid base64 data url (with optional media type and ;base64)
@@ -233,16 +232,10 @@ def scale_coordinates_with_padding(
233232
scaled_x = x * scale_factor + pad_left
234233
scaled_y = y * scale_factor + pad_top
235234

236-
if (
237-
scaled_x < 0
238-
or scaled_y < 0
239-
or scaled_x > max_width
240-
or scaled_y > max_height
241-
):
235+
if scaled_x < 0 or scaled_y < 0 or scaled_x > max_width or scaled_y > max_height:
242236
error_msg = "Coordinates are outside the padded image area"
243237
raise ValueError(error_msg)
244238

245-
246239
return scaled_x, scaled_y
247240

248241

0 commit comments

Comments
 (0)