From 024340962b454c5e1cb99f0807fa5d64caa5adcd Mon Sep 17 00:00:00 2001 From: vyokky <7678676@qq.com> Date: Wed, 11 Jun 2025 14:30:01 +0800 Subject: [PATCH 1/5] readme updated --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d8bfae52..bd573df3c 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ [![YouTube](https://img.shields.io/badge/YouTube-white?logo=youtube&logoColor=%23FF0000)](https://www.youtube.com/watch?v=QT_OhygMVXU)  - +microsoft%2FUFO | Trendshift From 24e24a68d894c4ece8c948e9883d094824506a81 Mon Sep 17 00:00:00 2001 From: vyokky <7678676@qq.com> Date: Wed, 11 Jun 2025 20:33:22 +0800 Subject: [PATCH 2/5] add excel new APIs --- ufo/automator/app_apis/excel/excelclient.py | 142 +++++++++++++++++++- ufo/prompts/apps/excel/api.yaml | 37 +++++ 2 files changed, 177 insertions(+), 2 deletions(-) diff --git a/ufo/automator/app_apis/excel/excelclient.py b/ufo/automator/app_apis/excel/excelclient.py index b65d2511e..d6b74e046 100644 --- a/ufo/automator/app_apis/excel/excelclient.py +++ b/ufo/automator/app_apis/excel/excelclient.py @@ -2,7 +2,7 @@ # Licensed under the MIT License. import os -from typing import Any, Dict, List, Type, Union +from typing import Any, Dict, List, Optional, Type, Union import pandas as pd @@ -171,7 +171,9 @@ def reorder_columns(self, sheet_name: str, desired_order: List[str] = None) -> s insert_offset = 1 for name, data in column_data: - insert_pos = self.get_nth_non_empty_position(insert_offset, empty_columns) + insert_pos = self.get_nth_non_empty_position( + insert_offset, empty_columns + ) print(f"✅ Inserting '{name}' at position {insert_pos}") for row_index, value in enumerate(data, start=1): ws.Cells(row_index, insert_pos).Value = value @@ -241,6 +243,88 @@ def get_range_values( return [list(row) for row in values] + def set_cell_value( + self, + sheet_name: str, + row: int, + col: Union[int, str], + value: Optional[Any] = None, + is_formula: bool = False, + ) -> str: + """ + Set the value of a cell in the specified sheet. + :param sheet_name: The name of the sheet. + :param row: The row number (1-based). + :param col: The column number (1-based) or letter (e.g., 'A'). + :param value: The value to set in the cell. If None, just select the cell. + :param is_formula: If True, treat the value as a formula. + :return: Success message or error message. + """ + sheet_list = [sheet.Name for sheet in self.com_object.Sheets] + if sheet_name not in sheet_list: + print( + f"Sheet {sheet_name} not found in the workbook, using the first sheet." + ) + sheet_name = 1 + + sheet = self.com_object.Sheets(sheet_name) + + if isinstance(col, str): + col = self.letters_to_number(col) + + if value is None: + sheet.Cells(row, col).Select() + return f"Cell {row}:{col} is selected. No value set." + + try: + if is_formula: + sheet.Cells(row, col).Formula = value + else: + sheet.Cells(row, col).Value = value + return f"Cell {row}:{col} set to '{value}'." + except Exception as e: + return f"Failed to set cell {row}:{col}. Error: {e}" + + def auto_fill( + self, + sheet_name: str, + start_row: int, + start_col: Union[int, str], + end_row: int, + end_col: Union[int, str], + ) -> str: + """ + Auto-fill a range of cells in the specified sheet. + :param sheet_name: The name of the sheet. + :param start_row: The starting row number (1-based). + :param start_col: The starting column number (1-based) or letter (e.g., 'A'). + :param end_row: The ending row number (1-based). + :param end_col: The ending column number (1-based) or letter (e.g., 'A'). + :return: Success message or error message. + """ + sheet_list = [sheet.Name for sheet in self.com_object.Sheets] + if sheet_name not in sheet_list: + print( + f"Sheet {sheet_name} not found in the workbook, using the first sheet." + ) + sheet_name = 1 + + sheet = self.com_object.Sheets(sheet_name) + + if isinstance(start_col, str): + start_col = self.letters_to_number(start_col) + if isinstance(end_col, str): + end_col = self.letters_to_number(end_col) + + try: + start_cell = sheet.Cells(start_row, start_col) + end_cell = sheet.Cells(end_row, end_col) + fill_range = sheet.Range(start_cell, end_cell) + fill_range.FillDown() + return f"Auto-filled range from {start_row}:{start_col} to {end_row}:{end_col}." + except Exception as e: + return f"Failed to auto-fill range {start_row}:{start_col} to {end_row}:{end_col}. Error: {e}" + def save_as( self, file_dir: str = "", file_name: str = "", file_ext: str = "" ) -> None: @@ -459,6 +543,60 @@ def name(cls) -> str: return "reorder_columns" +@ExcelWinCOMReceiver.register +class SetCellValueCommand(WinCOMCommand): + """ + The command to set the value of a cell. + """ + + def execute(self): + """ + Execute the command to set the value of a cell. + :return: The result of setting the cell value. + """ + return self.receiver.set_cell_value( + sheet_name=self.params.get("sheet_name", 1), + row=self.params.get("row", 1), + col=self.params.get("col", 1), + value=self.params.get("value", None), + is_formula=self.params.get("is_formula", False), + ) + + @classmethod + def name(cls) -> str: + """ + The name of the command. + """ + return "set_cell_value" + + +@ExcelWinCOMReceiver.register +class AutoFillCommand(WinCOMCommand): + """ + The command to auto-fill a range of cells. + """ + + def execute(self): + """ + Execute the command to auto-fill a range of cells. + :return: The result of auto-filling the range. + """ + return self.receiver.auto_fill( + sheet_name=self.params.get("sheet_name", 1), + start_row=self.params.get("start_row", 1), + start_col=self.params.get("start_col", 1), + end_row=self.params.get("end_row", 1), + end_col=self.params.get("end_col", 1), + ) + + @classmethod + def name(cls) -> str: + """ + The name of the command. + """ + return "auto_fill" + + @ExcelWinCOMReceiver.register class SaveAsCommand(WinCOMCommand): """ diff --git a/ufo/prompts/apps/excel/api.yaml b/ufo/prompts/apps/excel/api.yaml index 424a419bf..8575df1d8 100644 --- a/ufo/prompts/apps/excel/api.yaml +++ b/ufo/prompts/apps/excel/api.yaml @@ -45,6 +45,43 @@ select_table_range: [4] Available control item: Any control item in the Excel app. [5] Return: A message indicating whether the range is selected successfully or not. + +set_cell_value: + summary: |- + "set_cell_value" is to set the value of a cell in the sheet of the Excel app. + class_name: |- + SetCellValueCommand + usage: |- + [1] API call: set_cell_value(sheet_name: str, row: int, col: Union[int, str], value: Optional[Any] = None, is_formula: bool = False) + [2] Args: + - sheet_name: The name of the sheet. + - row: The row number (1-based). + - col: The column number (1-based) or letter (e.g., 'A'). + - value: The value to set in the cell. If None, just select the cell. + - is_formula: If True, treat the value as a formula, otherwise treat it as a normal value. + [3] Example: set_cell_value(sheet_name="Sheet1", row=1, col=1, value="Hello", is_formula=False), set_cell_value(sheet_name="Sheet1", row=2, col="B", value="=SUM(A1:A10)", is_formula=True) + [4] Available control item: Any control item in the Excel app. + [5] Return: A message indicating whether the cell value is set successfully or not. + +auto_fill: + summary: |- + "auto_fill" is to auto-fill a range of cells in the sheet of the Excel app. This can apply to a range of cells that have a pattern, such as dates, numbers, formulas, etc. + class_name: |- + AutoFillCommand + usage: |- + [1] API call: auto_fill(sheet_name: str, start_row: int, start_col: Union[int, str], end_row: int, end_col: Union[int, str]) + [2] Args: + - sheet_name: The name of the sheet. + - start_row: The starting row number (1-based). + - start_col: The starting column number (1-based) or letter (e.g., 'A'). + - end_row: The ending row number (1-based). + - end_col: The ending column number (1-based) or letter (e.g., 'A'). + [3] Example: auto_fill(sheet_name="Sheet1", start_row=1, start_col=1, end_row=10, end_col=3) + [4] Available control item: Any control item in the Excel app. + [5] Return: A message indicating whether the range is auto-filled successfully or not. + + + save_as: summary: |- "save_as" is a shortcut and quickest way to save or export the Excel document to a specified file format with one command. From 35abe312882bd52ef05d18e4aec72c4fe6a9da7f Mon Sep 17 00:00:00 2001 From: vyokky <7678676@qq.com> Date: Tue, 5 Aug 2025 22:15:27 +0800 Subject: [PATCH 3/5] disclaimer --- DISCLAIMER.md | 3 +++ ufo/config/config_dev.yaml | 2 +- ufo/llm/openai.py | 12 ++++++++++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/DISCLAIMER.md b/DISCLAIMER.md index 1688b3dbc..a35936c99 100644 --- a/DISCLAIMER.md +++ b/DISCLAIMER.md @@ -2,6 +2,9 @@ By choosing to run the provided code, you acknowledge and agree to the following terms and conditions regarding the functionality and data handling practices: +## Purpose +This repository is intended solely for research purposes. The code provided herein is not designed, tested, or validated for third-party production use. Users are expected to exercise their own judgment and due diligence when utilizing any part of this codebase. Microsoft is committed to building Responsible and Trustworthy AI. To learn more about our principles and practices, please refer to our [principles and approach](https://www.microsoft.com/en-us/ai/principles-and-approach). + ## 1. Code Functionality: The code you are about to execute has the capability to capture screenshots of your working desktop environment and active applications. These screenshots will be processed and sent to the GPT model for inference. diff --git a/ufo/config/config_dev.yaml b/ufo/config/config_dev.yaml index f65d72dd1..a083e295c 100644 --- a/ufo/config/config_dev.yaml +++ b/ufo/config/config_dev.yaml @@ -17,7 +17,7 @@ MAXIMIZE_WINDOW: False # Whether to maximize the application window before the JSON_PARSING_RETRY: 3 # The retry times for the json parsing SAFE_GUARD: True # Whether to use the safe guard to prevent the model from doing sensitve operations. -CONTROL_LIST: ["Button", "Edit", "TabItem", "Document", "ListItem", "MenuItem", "ScrollBar", "TreeItem", "Hyperlink", "ComboBox", "RadioButton", "DataItem", "Spinner", "CheckBox"] +CONTROL_LIST: ["Button", "Edit", "TabItem", "Document", "ListItem", "MenuItem", "ScrollBar", "TreeItem", "Hyperlink", "ComboBox", "RadioButton", "Image", "Spinner", "CheckBox"] # The list of widgets that allowed to be selected, in uia backend, it will be used for filter the control_type, while in win32 backend, it will be used for filter the class_name. HISTORY_KEYS: ["Step", "Subtask", "Action", "UserConfirm"] # The keys of the action history for the next step. diff --git a/ufo/llm/openai.py b/ufo/llm/openai.py index ef5736238..b18b9b14b 100644 --- a/ufo/llm/openai.py +++ b/ufo/llm/openai.py @@ -18,7 +18,9 @@ class BaseOpenAIService(BaseService): - def __init__(self, config: Dict[str, Any], agent_type: str, api_provider: str, api_base: str) -> None: + def __init__( + self, config: Dict[str, Any], agent_type: str, api_provider: str, api_base: str + ) -> None: """ Create an OpenAI service instance. :param config: The configuration for the OpenAI service. @@ -437,6 +439,7 @@ def load_auth_record() -> Optional[AuthenticationRecord]: print("failed to acquire token from AAD for OpenAI", e) raise e + class OpenAIService(BaseOpenAIService): """ The OpenAI service class to interact with the OpenAI API. @@ -448,7 +451,12 @@ def __init__(self, config: Dict[str, Any], agent_type: str) -> None: :param config: The configuration for the OpenAI service. :param agent_type: The type of the agent. """ - super().__init__(config, agent_type, config[agent_type]["API_TYPE"].lower(), config[agent_type]["API_BASE"]) + super().__init__( + config, + agent_type, + config[agent_type]["API_TYPE"].lower(), + config[agent_type]["API_BASE"], + ) def chat_completion( self, From e1cb81cc2f155c7a4372bdd88c6631a650e10b8f Mon Sep 17 00:00:00 2001 From: vyokky <7678676@qq.com> Date: Wed, 6 Aug 2025 13:05:22 +0800 Subject: [PATCH 4/5] disclaimer --- README.md | 3 +++ documents/docs/index.md | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index bd573df3c..0e3ef62d2 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,9 @@ The UFO² team is actively working on the following features and improvements: ## ⚠️ Disclaimer By choosing to run the provided code, you acknowledge and agree to the following terms and conditions regarding the functionality and data handling practices in [DISCLAIMER.md](./DISCLAIMER.md) +This repository is intended solely for research purposes. The code provided herein is not designed, tested, or validated for third-party production use. Users are expected to exercise their own judgment and due diligence when utilizing any part of this codebase. Microsoft is committed to building Responsible and Trustworthy AI. To learn more about our principles and practices, please refer to our [principles and approach](https://www.microsoft.com/en-us/ai/principles-and-approach). + + ## logo Trademarks diff --git a/documents/docs/index.md b/documents/docs/index.md index cd4307b28..ab6e305d3 100644 --- a/documents/docs/index.md +++ b/documents/docs/index.md @@ -53,6 +53,10 @@ For a deep dive see our [technical report](https://arxiv.org/abs/2504.14603). ## 🚀 Quick Start Please follow the [Quick Start Guide](./getting_started/quick_start.md) to get started with UFO. +!!! note + This repository is intended solely for research purposes. The code provided herein is not designed, tested, or validated for third-party production use. Users are expected to exercise their own judgment and due diligence when utilizing any part of this codebase. Microsoft is committed to building Responsible and Trustworthy AI. To learn more about our principles and practices, please refer to our [principles and approach](https://www.microsoft.com/en-us/ai/principles-and-approach). + + ## 🌐 Media Coverage From 58ebb4cd148dfa5f8037bdc9344a63ebc9f567ae Mon Sep 17 00:00:00 2001 From: S Sravandeep Reddy Date: Mon, 11 Aug 2025 23:22:33 +0530 Subject: [PATCH 5/5] Update config_dev.yaml --- ufo/config/config_dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ufo/config/config_dev.yaml b/ufo/config/config_dev.yaml index a083e295c..e409aaca5 100644 --- a/ufo/config/config_dev.yaml +++ b/ufo/config/config_dev.yaml @@ -46,7 +46,7 @@ HOSTAGENT_PROMPT: "ufo/prompts/share/base/host_agent.yaml" # The prompt for the # Due to the limitation of input size, lite version of the prompt help users have a taste. And the path is "ufo/prompts/share/lite/host_agent.yaml" APPAGENT_PROMPT: "ufo/prompts/share/base/app_agent.yaml" # The prompt for the action selection # Lite version: "ufo/prompts/share/lite/app_agent.yaml" -FOLLOWERAHENT_PROMPT: "ufo/prompts/share/base/app_agent.yaml" # The prompt for the follower agent +FOLLOWERAGENT_PROMPT: "ufo/prompts/share/base/app_agent.yaml" # The prompt for the follower agent EVALUATION_PROMPT: "ufo/prompts/evaluation/evaluate.yaml" # The prompt for the evaluation