diff --git a/README.md b/README.md index 6758b9a6..2099d4ee 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,21 @@ Or add it to your own client manually: } ``` +### macOS image capture + +`get_app_state` and action tools return a PNG screenshot together with the accessibility tree. On macOS, screenshot size can be tuned with optional environment variables. Set them before starting the `open-computer-use` MCP host process. The macOS proxy forwards that process's current values on every request, so restarting the host with different values applies them without restarting the app agent; variables that are unset in the host use the defaults. + +| Variable | Default | Meaning | +| --- | --- | --- | +| `OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT` | `5` | Seconds to wait for ScreenCaptureKit before omitting the screenshot image content from the result. | +| `OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION` | `1280` | Integer long-edge pixel cap for the returned PNG. | +| `OPEN_COMPUTER_USE_IMAGE_MAX_BYTES` | `900000` | Best-effort byte budget for the encoded PNG. | +| `OPEN_COMPUTER_USE_IMAGE_MIN_SCALE` | `0.25` | Byte-budget retry multiplier applied after the dimension cap. For example, a 500 px capped long edge with `0.25` may retry down to 125 px; it never enlarges the PNG chosen by the dimension cap. | + +Invalid, non-finite, or non-positive values fall back to the defaults. `OPEN_COMPUTER_USE_IMAGE_MIN_SCALE` values above `1` also fall back. + +Coordinate tools keep using the actual returned PNG dimensions, so downsampling does not change click or drag mapping. + ### Skill Install the skill directly: diff --git a/README.zh-CN.md b/README.zh-CN.md index 1487d22c..2d63b639 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -76,6 +76,21 @@ ocu install-codex-mcp } ``` +### macOS 截图尺寸 + +`get_app_state` 和 action 类工具会和 accessibility tree 一起返回 PNG 截图。macOS 上可以通过可选环境变量调整截图尺寸。请在启动 `open-computer-use` MCP host 进程前设置这些变量。macOS proxy 会在每次请求时转发该进程的当前值，因此修改配置后只需重启 host，不需要重启 app agent；host 未设置的变量会使用默认值。 + +| 变量 | 默认值 | 作用 | +| --- | --- | --- | +| `OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT` | `5` | ScreenCaptureKit 等待秒数；超时会从结果里省略截图 image 内容。 | +| `OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION` | `1280` | 返回 PNG 的整数长边像素上限。 | +| `OPEN_COMPUTER_USE_IMAGE_MAX_BYTES` | `900000` | 编码后 PNG 的 best-effort 字节预算。 | +| `OPEN_COMPUTER_USE_IMAGE_MIN_SCALE` | `0.25` | 作为长边上限之后的字节预算重试倍率。例如长边已限制到 500 px，`0.25` 允许继续重试缩小到 125 px；它不会放大长边上限已经选出的 PNG。 | + +非法值、非有限数或非正数会回退到默认值。`OPEN_COMPUTER_USE_IMAGE_MIN_SCALE` 大于 `1` 时也会回退。 + +坐标类工具会读取实际返回 PNG 的尺寸做映射，因此降采样不会改变 click 或 drag 的坐标换算。 + ### Skill 一键安装skill： diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift index 04cdd75b..50ddaaef 100644 --- a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift +++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift @@ -6,6 +6,12 @@ import OpenComputerUseKit private let appAgentCommand = "__open-computer-use-app-agent" private let appAgentDisableEnvironmentKey = "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY" private let appAgentProcessStartDate = Date() +private let appAgentImageEnvironmentKeys: Set = [ + "OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT", + "OPEN_COMPUTER_USE_IMAGE_MAX_BYTES", + "OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION", + "OPEN_COMPUTER_USE_IMAGE_MIN_SCALE", +] enum MacOSAppAgentProxy { static func isAgentInvocation(arguments: [String]) -> Bool { @@ -116,6 +122,7 @@ enum MacOSAppAgentProxy { let response = try client.request([ "kind": "mcp", "line": line, + "environment": proxiedEnvironment(), ]) if let responseLine = response["response"] as? String { @@ -333,14 +340,17 @@ private final class AppAgentConnection: @unchecked Sendable { return ["ok": true] case "mcp": let line = request["line"] as? String ?? "" - if let response = server.handle(line: line) { + let environment = request["environment"] as? [String: String] ?? [:] + if let response = AppAgentEnvironment.withOverrides(environment, clearing: appAgentImageEnvironmentKeys, { + server.handle(line: line) + }) { return ["response": response] } return ["response": NSNull()] case "cli": let arguments = request["arguments"] as? [String] ?? [] let environment = request["environment"] as? [String: String] ?? [:] - let response = AppAgentEnvironment.withOverrides(environment) { + let response = AppAgentEnvironment.withOverrides(environment, clearing: appAgentImageEnvironmentKeys) { runCLI(arguments: arguments) } return [ @@ -410,19 +420,24 @@ private final class AppAgentConnection: @unchecked Sendable { private enum AppAgentEnvironment { private static let lock = NSLock() - static func withOverrides(_ overrides: [String: String], _ body: () throws -> T) rethrows -> T { - guard !overrides.isEmpty else { - return try body() - } - + static func withOverrides( + _ overrides: [String: String], + clearing keys: Set, + _ body: () throws -> T + ) rethrows -> T { lock.lock() defer { lock.unlock() } + var affectedKeys = keys + affectedKeys.formUnion(overrides.keys) let previousValues = Dictionary( - uniqueKeysWithValues: overrides.keys.map { key in - (key, ProcessInfo.processInfo.environment[key]) + uniqueKeysWithValues: affectedKeys.map { key in + (key, getenv(key).map { String(cString: $0) }) } ) + for key in keys where overrides[key] == nil { + unsetenv(key) + } for (key, value) in overrides { setenv(key, value, 1) } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1de04314..27477cad 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -128,7 +128,7 @@ - 开源版当前不复刻官方闭源实现里的 caller signing、私有 IPC、完整 overlay choreography 和 plugin 自安装逻辑。 - 因为官方 `SkyComputerUseClient` 带有宿主侧 launch constraints，普通 stdio MCP client 在本机上可能被系统直接杀掉；如果要探测官方 bundled `computer-use`，`scripts/computer-use-cli` 的 app-server 模式现在只适合做工具清单和协议面观察。官方 `1.0.755` 的真实 tool call 还会经过 service-side sender authorization / active IPC client 追踪，外部 raw helper 即使走已签名 Codex binary，也可能返回 `Sender process is not authenticated`；需要真实使用官方工具时应走正常 Codex agent/tool 调用链，开源版则继续提供可直连的 `open-computer-use` MCP server。 - 当前权限引导已经具备可运行 app、深链、拖拽辅助，以及一版更接近官方的 accessory panel 入场动画和返回 affordance；点击链路也已经补上独立 visual cursor、官方 asset fallback 和相对目标 window 的排序逻辑，并且在 overlay 可见期间会持续重申“排在目标 window 之上”，避免用户手动激活目标 app 后 cursor 被目标窗口重新盖住；但整体还没有完全复刻官方那套嵌入式 choreography / host 集成 / session approval 体验。 -- screenshot 当前通过 `ScreenCaptureKit` 捕获目标窗口，并以 MCP `image` content block 的 base64 PNG 返回，不再把普通 app 截图落盘到仓库或临时目录；编码前会按最大尺寸和目标字节数自适应缩小，避免复杂页面的大 PNG 触发 host 侧 MCP result 降级，同时 coordinate tools 继续按实际返回的 screenshot pixel 尺寸映射坐标；单次 ScreenCaptureKit capture 会设置超时，超时后省略 image block 而不是卡住整个 `get_app_state`。 +- screenshot 当前通过 `ScreenCaptureKit` 捕获目标窗口，并以 MCP `image` content block 的 base64 PNG 返回，不再把普通 app 截图落盘到仓库或临时目录；编码前会按最大尺寸和目标字节数自适应缩小，避免复杂页面的大 PNG 触发 host 侧 MCP result 降级，同时 coordinate tools 继续按实际返回的 screenshot pixel 尺寸映射坐标。macOS MCP proxy 会把当前 host 的 `OPEN_COMPUTER_USE_*` 环境逐条转发给 app agent，并在请求期间清除 host 未设置的 image capture key，避免继承常驻 agent 的值；截图的 capture timeout、长边像素上限、PNG 字节预算和字节预算继续缩小时的最小缩放比例可分别通过 `OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT`、`OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION`、`OPEN_COMPUTER_USE_IMAGE_MAX_BYTES`、`OPEN_COMPUTER_USE_IMAGE_MIN_SCALE` 调整；单次 ScreenCaptureKit capture 超时后省略 image block，而不是卡住整个 `get_app_state`。 - 会话状态现在是进程内内存态，保存每个 app 最近一次 snapshot 和 element index 映射。 ## 主要验证路径 diff --git a/docs/histories/2026-07/20260702-0126-configurable-image-capture.md b/docs/histories/2026-07/20260702-0126-configurable-image-capture.md new file mode 100644 index 00000000..17e1c20b --- /dev/null +++ b/docs/histories/2026-07/20260702-0126-configurable-image-capture.md @@ -0,0 +1,32 @@ +## [2026-07-02 01:26] | Task: configurable image capture + +### Execution Context +* **Agent ID**: `Amp` +* **Base Model**: `Amp deep mode, model not exposed` +* **Runtime**: `macOS SwiftPM` + +### User Query +> `get_app_state` 和动作工具返回的截图占用大量上下文，希望能按 MCP host 的预算调小图片，同时保持真实桌面操作和坐标映射安全。 + +### Changes Overview +**Scope:** macOS screenshot capture and app-agent proxy, README, architecture docs, release notes + +**Key Actions:** +- **[Config]**: macOS 截图捕获新增 `OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT`、`OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION`、`OPEN_COMPUTER_USE_IMAGE_MAX_BYTES`、`OPEN_COMPUTER_USE_IMAGE_MIN_SCALE` 四个环境变量。 +- **[Runtime]**: MCP proxy 把 host 的 `OPEN_COMPUTER_USE_*` 环境逐条转发给 app agent，并在请求期间清除 host 未设置的 image capture key、完成后恢复 agent 原环境；`WindowCapture` 每次捕获时读取当前配置，控制 ScreenCaptureKit 超时、PNG 长边上限、编码后字节预算和降采样比例下限。 +- **[Scaling]**: 修复 `maxDimension / nativeSize` 小于 `minScale` 时 resize 循环不执行、从而返回原图的问题；现在显式的长边上限会优先生效，`minScale` 只限制在该长边上限基础上按字节预算继续缩小时的下限。 +- **[Validation]**: 非法环境变量值会回退到默认值；返回 PNG 会遵守长边上限，按字节预算继续缩小时也会按实际返回 PNG 尺寸换算坐标。 +- **[Tests]**: 补充单元测试覆盖环境变量解析、非法配置回退、`minScale` 夹取行为和缩小后 PNG 尺寸到窗口坐标的换算。 +- **[Docs]**: 同步 README、中文 README、架构文档和功能发布记录。 + +### Design Intent (Why) +默认截图边界已经能避免一部分过大的 PNG，但不同 MCP host 对 image block 的上下文成本差异很大。把已有边界开放成环境变量可以保持默认兼容，同时让 Codex、Claude、Gemini 或其他 host 根据自己的预算调小图片。坐标类工具继续从返回 PNG 读取实际尺寸再映射回窗口坐标，因此降采样不会破坏 click / drag 的坐标语义。 + +### Files Modified +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift` +- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift` +- `apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift` +- `README.md` +- `README.zh-CN.md` +- `docs/ARCHITECTURE.md` +- `docs/releases/feature-release-notes.md` diff --git a/docs/releases/feature-release-notes.md b/docs/releases/feature-release-notes.md index f51fdce5..338524dd 100644 --- a/docs/releases/feature-release-notes.md +++ b/docs/releases/feature-release-notes.md @@ -5,6 +5,7 @@ | 日期 | 功能域 | 用户价值 | 变更摘要 | | --- | --- | --- | --- | | 2026-07-08 | 快照预算与长文本控制 | 长网页、长列表和复杂表格可以显式提高 accessibility tree 预算，读取长消息或文档时也能按需选择更大的文本上限或全文模式。 | 发布 `0.2.0`，三端默认 tree budget 统一为 1200/64，并为 `get_app_state` / `snapshot` 增加 `max_tree_nodes`、`max_tree_depth` 与 `text_limit` / `--text-limit`；`show_full_text` / `--show-full-text` 已由 `text_limit: "max"` / `--text-limit max` 替代。 | +| 2026-07-02 | macOS 截图上下文控制 | MCP host 可以按自己的上下文预算调小 `get_app_state` 和 action tool 返回的截图，降低复杂窗口反复返回大 PNG 对 agent 上下文的压力。 | macOS 截图捕获新增 `OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT`、`OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION`、`OPEN_COMPUTER_USE_IMAGE_MAX_BYTES`、`OPEN_COMPUTER_USE_IMAGE_MIN_SCALE` 配置；默认行为保持不变，并确保较小的 `OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION` 仍作为返回 PNG 的长边上限生效。 | ## 2026-06 diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift index 597c37ef..1cea07ea 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift @@ -1,6 +1,7 @@ import AppKit import ApplicationServices import CoreGraphics +import Darwin import Foundation import ScreenCaptureKit @@ -83,10 +84,6 @@ public struct SnapshotTextLimit: Equatable, Sendable { let accessibilityTreeMaxNodeCount = AccessibilityTreeLimits.defaultMaxNodeCount let accessibilityTreeMaxDepth = AccessibilityTreeLimits.defaultMaxDepth -let screenshotCaptureTimeout: TimeInterval = 5 -let screenshotResultMaxPNGBytes = 900_000 -let screenshotResultMaxDimension: CGFloat = 1280 -let screenshotResultMinScale: CGFloat = 0.25 private let windowVisibilityRecoveryDelay: TimeInterval = 0.7 private let axWebAreaRole = "AXWebArea" private let axContentsAttribute = "AXContents" @@ -94,6 +91,88 @@ private let axVisibleChildrenAttribute = "AXVisibleChildren" private let anonymousActionTargetMaxWidth: CGFloat = 240 private let anonymousActionTargetMaxHeight: CGFloat = 120 +struct ImageCaptureConfig: Sendable, Equatable { + let captureTimeout: TimeInterval + let maxPNGBytes: Int + let maxDimension: Int + let minScale: CGFloat + + static let defaults = ImageCaptureConfig( + captureTimeout: 5, + maxPNGBytes: 900_000, + maxDimension: 1280, + minScale: 0.25 + ) + + static var current: ImageCaptureConfig { + fromEnvironment([ + "OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT": liveEnvironmentValue("OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT"), + "OPEN_COMPUTER_USE_IMAGE_MAX_BYTES": liveEnvironmentValue("OPEN_COMPUTER_USE_IMAGE_MAX_BYTES"), + "OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION": liveEnvironmentValue("OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION"), + "OPEN_COMPUTER_USE_IMAGE_MIN_SCALE": liveEnvironmentValue("OPEN_COMPUTER_USE_IMAGE_MIN_SCALE"), + ].compactMapValues { $0 }) + } + + static func fromEnvironment(_ environment: [String: String] = ProcessInfo.processInfo.environment) -> ImageCaptureConfig { + ImageCaptureConfig( + captureTimeout: parseImageConfigPositiveDouble(environment["OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT"]) + ?? defaults.captureTimeout, + maxPNGBytes: parseImageConfigPositiveInt(environment["OPEN_COMPUTER_USE_IMAGE_MAX_BYTES"]) + ?? defaults.maxPNGBytes, + maxDimension: parseImageConfigPositiveInt(environment["OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION"]) + ?? defaults.maxDimension, + minScale: parseImageConfigUnitInterval(environment["OPEN_COMPUTER_USE_IMAGE_MIN_SCALE"]) + .map { CGFloat($0) } ?? defaults.minScale + ) + } +} + +private func liveEnvironmentValue(_ name: String) -> String? { + guard let value = getenv(name) else { + return nil + } + return String(cString: value) +} + +private func parseImageConfigPositiveInt(_ raw: String?) -> Int? { + guard let raw = raw?.trimmingCharacters(in: .whitespacesAndNewlines), + !raw.isEmpty, + let value = Int(raw), + value > 0 + else { + return nil + } + + return value +} + +private func parseImageConfigPositiveDouble(_ raw: String?) -> Double? { + guard let raw = raw?.trimmingCharacters(in: .whitespacesAndNewlines), + !raw.isEmpty, + let value = Double(raw), + value > 0, + value.isFinite + else { + return nil + } + + return value +} + +private func parseImageConfigUnitInterval(_ raw: String?) -> Double? { + guard let raw = raw?.trimmingCharacters(in: .whitespacesAndNewlines), + !raw.isEmpty, + let value = Double(raw), + value > 0, + value <= 1, + value.isFinite + else { + return nil + } + + return value +} + public struct AppSnapshot { public let app: RunningAppDescriptor public let windowTitle: String? @@ -413,6 +492,7 @@ private struct WindowCapture { let layer: Int let bounds: CGRect let image: CGImage? + let imageConfig: ImageCaptureConfig static func resolve(for pid: pid_t, titleHint: String?) -> WindowCapture? { guard let infoList = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] else { @@ -447,13 +527,14 @@ private struct WindowCapture { return nil } - let image = captureImage(windowID: best.windowID, bounds: best.bounds) + let imageConfig = ImageCaptureConfig.current + let image = captureImage(windowID: best.windowID, bounds: best.bounds, config: imageConfig) - return WindowCapture(windowID: best.windowID, layer: best.layer, bounds: best.bounds, image: image) + return WindowCapture(windowID: best.windowID, layer: best.layer, bounds: best.bounds, image: image, imageConfig: imageConfig) } - private static func captureImage(windowID: CGWindowID, bounds: CGRect) -> CGImage? { - try? BlockingAsyncBridge.run(timeout: screenshotCaptureTimeout) { + private static func captureImage(windowID: CGWindowID, bounds: CGRect, config: ImageCaptureConfig) -> CGImage? { + try? BlockingAsyncBridge.run(timeout: config.captureTimeout) { let shareableContent = try await SCShareableContent.current guard let window = shareableContent.windows.first(where: { $0.windowID == windowID }) else { return nil @@ -484,7 +565,12 @@ private struct WindowCapture { return nil } - return boundedScreenshotPNGData(for: image) + return boundedScreenshotPNGData( + for: image, + maxBytes: imageConfig.maxPNGBytes, + maxDimension: CGFloat(imageConfig.maxDimension), + minScale: imageConfig.minScale + ) } } @@ -531,24 +617,34 @@ func preferredWindowCaptureCandidate(_ candidates: [WindowCaptureCandidate], tit func boundedScreenshotPNGData( for image: CGImage, - maxBytes: Int = screenshotResultMaxPNGBytes, - maxDimension: CGFloat = screenshotResultMaxDimension, - minScale: CGFloat = screenshotResultMinScale + maxBytes: Int = ImageCaptureConfig.defaults.maxPNGBytes, + maxDimension: CGFloat = CGFloat(ImageCaptureConfig.defaults.maxDimension), + minScale: CGFloat = ImageCaptureConfig.defaults.minScale ) -> Data? { - guard image.width > 0, image.height > 0, maxBytes > 0 else { + guard image.width > 0, + image.height > 0, + maxBytes > 0, + maxDimension >= 1, + maxDimension.isFinite, + maxDimension.rounded(.down) == maxDimension, + minScale > 0, + minScale <= 1, + minScale.isFinite + else { return nil } let original = pngData(for: image) let largestDimension = CGFloat(max(image.width, image.height)) var scale = min(1, maxDimension / largestDimension) + let byteBudgetFloorScale = scale * minScale if scale >= 1, let original, original.count <= maxBytes { return original } var best = original - while scale >= minScale { + while scale >= byteBudgetFloorScale { guard let resized = resizedCGImage(image, scale: scale), let data = pngData(for: resized) else { @@ -559,6 +655,9 @@ func boundedScreenshotPNGData( if data.count <= maxBytes { return data } + if resized.width == 1, resized.height == 1 { + return data + } scale *= 0.85 } @@ -572,8 +671,8 @@ private func pngData(for image: CGImage) -> Data? { } private func resizedCGImage(_ image: CGImage, scale: CGFloat) -> CGImage? { - let width = max(1, Int((CGFloat(image.width) * scale).rounded())) - let height = max(1, Int((CGFloat(image.height) * scale).rounded())) + let width = max(1, Int((CGFloat(image.width) * scale).rounded(.down))) + let height = max(1, Int((CGFloat(image.height) * scale).rounded(.down))) let colorSpace = CGColorSpaceCreateDeviceRGB() let bitmapInfo = CGImageAlphaInfo.premultipliedLast.rawValue diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift index ca54cd15..f5d1b63b 100644 --- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift +++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift @@ -1,4 +1,5 @@ import AppKit +import Darwin import ImageIO import XCTest @testable import OpenComputerUseKit @@ -236,6 +237,263 @@ final class OpenComputerUseKitTests: XCTestCase { XCTAssertEqual(size.height, 24) } + func testImageCaptureConfigReadsValidEnvironmentOverrides() { + let config = ImageCaptureConfig.fromEnvironment([ + "OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT": " 2.5 ", + "OPEN_COMPUTER_USE_IMAGE_MAX_BYTES": " 120000 ", + "OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION": " 640 ", + "OPEN_COMPUTER_USE_IMAGE_MIN_SCALE": "0.1", + ]) + + XCTAssertEqual(config.captureTimeout, 2.5) + XCTAssertEqual(config.maxPNGBytes, 120_000) + XCTAssertEqual(config.maxDimension, 640) + XCTAssertEqual(config.minScale, 0.1) + } + + func testImageCaptureConfigCurrentReadsEnvironmentForEveryCapture() { + let key = "OPEN_COMPUTER_USE_IMAGE_MAX_BYTES" + let previousValue = getenv(key).map { String(cString: $0) } + defer { + if let previousValue { + setenv(key, previousValue, 1) + } else { + unsetenv(key) + } + } + + setenv(key, "120000", 1) + XCTAssertEqual(ImageCaptureConfig.current.maxPNGBytes, 120_000) + + setenv(key, "65000", 1) + XCTAssertEqual(ImageCaptureConfig.current.maxPNGBytes, 65_000) + } + + func testImageCaptureConfigFallsBackForInvalidEnvironmentOverrides() { + let config = ImageCaptureConfig.fromEnvironment([ + "OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT": "0", + "OPEN_COMPUTER_USE_IMAGE_MAX_BYTES": "nope", + "OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION": "0", + "OPEN_COMPUTER_USE_IMAGE_MIN_SCALE": "0", + ]) + + XCTAssertEqual(config, ImageCaptureConfig.defaults) + } + + func testImageCaptureConfigFallsBackForBlankAndNonFiniteOverrides() { + let config = ImageCaptureConfig.fromEnvironment([ + "OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT": "nan", + "OPEN_COMPUTER_USE_IMAGE_MAX_BYTES": " ", + "OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION": "inf", + "OPEN_COMPUTER_USE_IMAGE_MIN_SCALE": "nan", + ]) + + XCTAssertEqual(config, ImageCaptureConfig.defaults) + } + + func testImageCaptureConfigAcceptsUnitMinScaleBoundary() { + let config = ImageCaptureConfig.fromEnvironment([ + "OPEN_COMPUTER_USE_IMAGE_MIN_SCALE": "1", + ]) + + XCTAssertEqual(config.minScale, 1) + } + + func testImageCaptureConfigFallsBackForMinScaleAboveUnitBoundary() { + let config = ImageCaptureConfig.fromEnvironment([ + "OPEN_COMPUTER_USE_IMAGE_CAPTURE_TIMEOUT": "-1", + "OPEN_COMPUTER_USE_IMAGE_MIN_SCALE": "1.01", + ]) + + XCTAssertEqual(config.captureTimeout, ImageCaptureConfig.defaults.captureTimeout) + XCTAssertEqual(config.minScale, ImageCaptureConfig.defaults.minScale) + } + + func testImageCaptureConfigFallsBackForNegativeMinScale() { + let config = ImageCaptureConfig.fromEnvironment([ + "OPEN_COMPUTER_USE_IMAGE_MIN_SCALE": "-0.1", + ]) + + XCTAssertEqual(config.minScale, ImageCaptureConfig.defaults.minScale) + } + + func testImageCaptureConfigFallsBackForFractionalDimensionAndIntegerOverflow() { + let config = ImageCaptureConfig.fromEnvironment([ + "OPEN_COMPUTER_USE_IMAGE_MAX_BYTES": "92233720368547758070", + "OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION": "80.6", + ]) + + XCTAssertEqual(config.maxPNGBytes, ImageCaptureConfig.defaults.maxPNGBytes) + XCTAssertEqual(config.maxDimension, ImageCaptureConfig.defaults.maxDimension) + } + + func testImageCaptureConfigFallsBackForUnicodeDigits() { + let config = ImageCaptureConfig.fromEnvironment([ + "OPEN_COMPUTER_USE_IMAGE_MAX_BYTES": "１２３", + "OPEN_COMPUTER_USE_IMAGE_MAX_DIMENSION": "４８０", + ]) + + XCTAssertEqual(config.maxPNGBytes, ImageCaptureConfig.defaults.maxPNGBytes) + XCTAssertEqual(config.maxDimension, ImageCaptureConfig.defaults.maxDimension) + } + + func testBoundedScreenshotPNGDataHonorsMaxDimensionBelowMinScale() throws { + let image = try makeNoisyTestImage(width: 800, height: 600) + let data = try XCTUnwrap(boundedScreenshotPNGData( + for: image, + maxBytes: 1_000_000, + maxDimension: 80, + minScale: 0.25 + )) + let size = try imageSize(in: data) + + XCTAssertEqual(size.width, 80) + XCTAssertEqual(size.height, 60) + } + + func testBoundedScreenshotPNGDataRejectsNonPositiveMaxDimension() throws { + let image = try makeNoisyTestImage(width: 800, height: 600) + + XCTAssertNil(boundedScreenshotPNGData( + for: image, + maxBytes: 1_000_000, + maxDimension: 0, + minScale: 0.05 + )) + XCTAssertNil(boundedScreenshotPNGData( + for: image, + maxBytes: 1_000_000, + maxDimension: -1, + minScale: 0.05 + )) + } + + func testBoundedScreenshotPNGDataRejectsFractionalMaxDimension() throws { + let image = try makeNoisyTestImage(width: 800, height: 600) + + XCTAssertNil(boundedScreenshotPNGData( + for: image, + maxBytes: 1_000_000, + maxDimension: 80.6, + minScale: 0.05 + )) + } + + func testBoundedScreenshotPNGDataHonorsMaxDimensionOneBoundary() throws { + let image = try makeNoisyTestImage(width: 800, height: 600) + let data = try XCTUnwrap(boundedScreenshotPNGData( + for: image, + maxBytes: 1_000_000, + maxDimension: 1, + minScale: 0.05 + )) + let size = try imageSize(in: data) + + XCTAssertEqual(size.width, 1) + XCTAssertEqual(size.height, 1) + } + + func testBoundedScreenshotByteBudgetRetriesBelowDimensionCapWhenMinScaleAllows() throws { + let image = try makeNoisyTestImage(width: 800, height: 600) + let data = try XCTUnwrap(boundedScreenshotPNGData( + for: image, + maxBytes: 1, + maxDimension: 80, + minScale: 0.25 + )) + let size = try imageSize(in: data) + let point = screenshotPixelToWindowPoint( + CGPoint(x: CGFloat(size.width) / 2, y: CGFloat(size.height) / 2), + screenshotPixelSize: CGSize(width: size.width, height: size.height), + windowBounds: CGRect(x: 0, y: 0, width: 800, height: 600) + ) + + XCTAssertLessThan(max(size.width, size.height), 80) + XCTAssertGreaterThanOrEqual(max(size.width, size.height), 20) + XCTAssertEqual(point.x, 400, accuracy: 0.0001) + XCTAssertEqual(point.y, 300, accuracy: 0.0001) + } + + func testBoundedScreenshotPNGDataRejectsInvalidMinScale() throws { + let image = try makeNoisyTestImage(width: 800, height: 600) + + XCTAssertNil(boundedScreenshotPNGData( + for: image, + maxBytes: 1_000_000, + maxDimension: 80, + minScale: 1.01 + )) + } + + func testBoundedScreenshotPNGDataRejectsNonFiniteMinScale() throws { + let image = try makeNoisyTestImage(width: 800, height: 600) + + XCTAssertNil(boundedScreenshotPNGData( + for: image, + maxBytes: 1_000_000, + maxDimension: 80, + minScale: .nan + )) + } + + func testBoundedScreenshotPNGDataStopsAtOnePixelForSubnormalMinScale() throws { + let image = try makeNoisyTestImage(width: 32, height: 24) + let config = ImageCaptureConfig.fromEnvironment([ + "OPEN_COMPUTER_USE_IMAGE_MIN_SCALE": "5e-324", + ]) + let data = try XCTUnwrap(boundedScreenshotPNGData( + for: image, + maxBytes: 1, + maxDimension: 32, + minScale: config.minScale + )) + let size = try imageSize(in: data) + + XCTAssertGreaterThan(config.minScale, 0) + XCTAssertEqual(size.width, 1) + XCTAssertEqual(size.height, 1) + } + + func testBoundedScreenshotDimensionsMapBackToWindowCoordinatesBelowMinScale() throws { + let image = try makeNoisyTestImage(width: 800, height: 600) + let data = try XCTUnwrap(boundedScreenshotPNGData( + for: image, + maxBytes: 1_000_000, + maxDimension: 80, + minScale: 0.25 + )) + let size = try imageSize(in: data) + let point = screenshotPixelToWindowPoint( + CGPoint(x: CGFloat(size.width) / 2, y: CGFloat(size.height) / 2), + screenshotPixelSize: CGSize(width: size.width, height: size.height), + windowBounds: CGRect(x: 0, y: 0, width: 800, height: 600) + ) + + XCTAssertEqual(point.x, 400, accuracy: 0.0001) + XCTAssertEqual(point.y, 300, accuracy: 0.0001) + } + + func testBoundedScreenshotByteBudgetRetryCoordinatesUseReturnedDimensions() throws { + let image = try makeNoisyTestImage(width: 800, height: 600) + let data = try XCTUnwrap(boundedScreenshotPNGData( + for: image, + maxBytes: 50_000, + maxDimension: 800, + minScale: 0.05 + )) + let size = try imageSize(in: data) + let point = screenshotPixelToWindowPoint( + CGPoint(x: CGFloat(size.width) / 2, y: CGFloat(size.height) / 2), + screenshotPixelSize: CGSize(width: size.width, height: size.height), + windowBounds: CGRect(x: 0, y: 0, width: 800, height: 600) + ) + + XCTAssertLessThanOrEqual(data.count, 50_000) + XCTAssertLessThan(max(size.width, size.height), 800) + XCTAssertEqual(point.x, 400, accuracy: 0.0001) + XCTAssertEqual(point.y, 300, accuracy: 0.0001) + } + func testToolDefinitionCount() { XCTAssertEqual(ToolDefinitions.all.count, 9) }