From b26a8a136b18bad82bf728abc5cf548ab89099d5 Mon Sep 17 00:00:00 2001 From: karan-sanskar Date: Mon, 18 Aug 2025 00:15:13 +0530 Subject: [PATCH 1/5] [Chore]: Added proper error validations --- benchmate/api/sync.py | 189 ++++++++++++++++++++++++++---------------- 1 file changed, 117 insertions(+), 72 deletions(-) diff --git a/benchmate/api/sync.py b/benchmate/api/sync.py index 0e9adb5..c89c403 100644 --- a/benchmate/api/sync.py +++ b/benchmate/api/sync.py @@ -23,16 +23,19 @@ def sync(): # ------------------------------------------------------------- # ? Utility functions # ------------------------------------------------------------- -def run_cmd(cmd: str, cwd: Path | None = None) -> str | None: +def run_cmd(cmd: str, cwd: Path | None = None) -> tuple[str | None, str | None]: """ Execute a shell command and return stdout as string. - Returns None if the command fails. + Returns (stdout, None) if success. + Returns (None, "command: error_message") if the command fails. """ try: - return subprocess.check_output(cmd, cwd=cwd, shell=True, text=True, stderr=subprocess.STDOUT).strip() + out = subprocess.check_output(cmd, cwd=cwd, shell=True, text=True, stderr=subprocess.STDOUT).strip() + return out, None except subprocess.CalledProcessError as e: + error_message = e.output.strip() if e.output else str(e) frappe.log_error(f"Command failed: {cmd}\n{e.output}", "BenchMate run_cmd") - return None + return None, error_message def get_git_remote(app_path: Path) -> str | None: @@ -43,7 +46,7 @@ def get_git_remote(app_path: Path) -> str | None: if not git_config.exists(): return None - config = configparser.ConfigParser() + config = configparser.ConfigParser(strict=False) # allow duplicate options try: config.read(git_config) if config.has_section('remote "upstream"'): @@ -85,13 +88,11 @@ def _parse_pyproject_name(pyproject_path: Path) -> str | None: if not pyproject_path.exists(): return None try: - # ? Python 3.11+ has tomllib in stdlib try: import tomllib # type: ignore data = tomllib.loads(pyproject_path.read_text(encoding="utf-8")) except Exception: - # ? Fallback: very naive parsing for "name =" raw = pyproject_path.read_text(encoding="utf-8") for line in raw.splitlines(): if line.strip().startswith("name"): @@ -100,7 +101,6 @@ def _parse_pyproject_name(pyproject_path: Path) -> str | None: return val return None - # ? PEP 621 & poetry fallback if "project" in data and isinstance(data["project"], dict): return data["project"].get("name") if "tool" in data and isinstance(data["tool"], dict): @@ -120,99 +120,139 @@ def get_app_title(app_path: Path, app_name: str) -> str: 2. pyproject.toml (project.name / poetry.name) 3. Fallback: prettified app_name """ - # ? hooks.py hooks_title = _parse_hooks_title(app_path / app_name / "hooks.py") if hooks_title: return hooks_title - # ? pyproject.toml pyproject_name = _parse_pyproject_name(app_path / "pyproject.toml") if pyproject_name: return pyproject_name.replace("-", " ").replace("_", " ").title() - # ? fallback: format folder name return app_name.replace("-", " ").replace("_", " ").title() # ------------------------------------------------------------- # ? Bench & site parsing # ------------------------------------------------------------- -def parse_installed_apps(entry: Path) -> tuple[dict, str | None, str | None]: - """ - Parse bench-wide installed apps via `bench version --format json`. - Returns: - - installed_apps dict - - frappe_version - - frappe_branch - """ +def _robust_load_json_array(raw: str): + try: + return json.loads(raw) + except Exception: + pass + + start = raw.find("[") + end = raw.rfind("]") + if start != -1 and end != -1 and end > start: + try: + sub = raw[start : end + 1] + return json.loads(sub) + except Exception: + pass + + try: + return ast.literal_eval(raw) + except Exception: + raise ValueError("Unable to parse bench version output as JSON/pyliteral") + + +def parse_installed_apps(entry: Path) -> tuple[dict, str | None, str | None, str | None]: installed_apps = {} - frappe_version, frappe_branch = None, None + frappe_version, frappe_branch, error_message = None, None, None + + version_json, err = run_cmd("bench version --format json", cwd=entry) + if err: + return installed_apps, None, None, f"{entry} - bench version --format json - {err}" - version_json = run_cmd("bench version --format json", cwd=entry) if not version_json: - return installed_apps, None, None + return installed_apps, None, None, None + + try: + apps_data = _robust_load_json_array(version_json) + except Exception as e: + frappe.log_error(f"bench version returned non-JSON output for {entry}: {e}", "BenchMate Sync") + return installed_apps, None, None, None try: - apps_data = json.loads(version_json) - if isinstance(apps_data, list): - for app in apps_data: - app_name = app.get("app") - app_branch = app.get("branch") - app_version = app.get("version") - app_commit = app.get("commit") - - app_path = entry / "apps" / app_name - app_repo = get_git_remote(app_path) - app_title = get_app_title(app_path, app_name) - - installed_apps[app_name] = { - "title": app_title, - "branch": app_branch, - "version": app_version, - "commit": app_commit, - "repo": app_repo, - } - - # ? store frappe version/branch separately - if app_name == "frappe": - frappe_version, frappe_branch = app_version, app_branch + if isinstance(apps_data, dict): + found_list = None + for v in apps_data.values(): + if isinstance(v, list): + found_list = v + break + apps = found_list or [] + elif isinstance(apps_data, list | tuple): + apps = list(apps_data) + else: + apps = [] + except Exception: + apps = [] + + try: + for app in apps: + if not isinstance(app, dict): + continue + app_name = app.get("app") + if not app_name: + continue + app_branch = app.get("branch") + app_version = app.get("version") + app_commit = app.get("commit") + + app_path = entry / "apps" / app_name + app_repo = get_git_remote(app_path) + app_title = get_app_title(app_path, app_name) + + installed_apps[app_name] = { + "title": app_title, + "branch": app_branch, + "version": app_version, + "commit": app_commit, + "repo": app_repo, + } + + if app_name == "frappe": + frappe_version, frappe_branch = app_version, app_branch except Exception as e: - frappe.log_error(f"Failed to parse bench version json: {e}", "BenchMate Sync") + frappe.log_error(f"Unexpected error while processing bench apps for {entry}: {e}", "BenchMate Sync") - return installed_apps, frappe_version, frappe_branch + return installed_apps, frappe_version, frappe_branch, error_message -def get_site_apps(bench_path: Path, site_name: str, bench_apps: dict) -> dict: - """ - Get installed apps for a specific site using: - `bench --site list-apps --format json` - Filter against bench-wide apps metadata. - """ +def get_site_apps(bench_path: Path, site_name: str, bench_apps: dict) -> tuple[dict, str | None]: cmd = f"bench --site {site_name} list-apps --format json" - result = run_cmd(cmd, cwd=bench_path) + result, err = run_cmd(cmd, cwd=bench_path) + + if err: + return {}, f"{bench_path} - {cmd} - {err}" + if not result: - return {} + return {}, None site_apps = {} try: - data = json.loads(result) - for app_name in data.get(site_name, []): + # try robust parser (handles garbage around JSON) + data = _robust_load_json_array(result) + + if isinstance(data, dict): + app_list = data.get(site_name) or next((v for v in data.values() if isinstance(v, list)), []) + elif isinstance(data, list): + app_list = data + else: + app_list = [] + + for app_name in app_list or []: if app_name in bench_apps: site_apps[app_name] = bench_apps[app_name] + except Exception as e: - frappe.log_error(f"Failed to parse site apps for {site_name}: {e}", "BenchMate Sync") + frappe.log_error( + f"{cmd} returned invalid JSON for site {site_name}: {e}\nRaw Output:\n{result}", "BenchMate Sync" + ) - return site_apps + return site_apps, None def get_all_benches(default_path: str): - """ - Return list of all valid benches under default_path. - Each bench includes: - - frappe version & branch - - installed apps metadata - - detailed sites with site-specific installed apps - """ benches: list[dict] = [] root = Path(default_path).expanduser().resolve() @@ -227,18 +267,22 @@ def get_all_benches(default_path: str): sites_path = entry / "sites" procfile_path = entry / "Procfile" - # ? validate bench structure if not (sites_path.is_dir() and procfile_path.is_file()): continue - # ? Bench apps & frappe version/branch - bench_apps, frappe_version, frappe_branch = parse_installed_apps(entry) + is_error = False + error_message = None + + bench_apps, frappe_version, frappe_branch, err = parse_installed_apps(entry) + if err: + is_error, error_message = True, err - # ? Sites metadata sites = [] for s in sites_path.iterdir(): if s.is_dir() and s.name != "assets": - site_apps = get_site_apps(entry, s.name, bench_apps) + site_apps, site_err = get_site_apps(entry, s.name, bench_apps) + if site_err and not error_message: + is_error, error_message = True, site_err sites.append( { "site_name": s.name, @@ -248,7 +292,6 @@ def get_all_benches(default_path: str): } ) - # ? final bench info benches.append( { "name": entry.name, @@ -257,6 +300,8 @@ def get_all_benches(default_path: str): "branch": frappe_branch, "sites": sites, "installed_apps": bench_apps, + "is_error": is_error, + "error_message": error_message, } ) From 1033f832a5ef823098e6ed5045b87433e6c11bf0 Mon Sep 17 00:00:00 2001 From: karan-sanskar Date: Mon, 18 Aug 2025 00:20:26 +0530 Subject: [PATCH 2/5] [Chore]: Added proper error validations --- benchmate/api/sync.py | 122 +++++++++++++++++++++++++++++++++++------- 1 file changed, 102 insertions(+), 20 deletions(-) diff --git a/benchmate/api/sync.py b/benchmate/api/sync.py index c89c403..6c1d7ab 100644 --- a/benchmate/api/sync.py +++ b/benchmate/api/sync.py @@ -12,22 +12,30 @@ @frappe.whitelist() def sync(): """ - Sync and return list of all valid benches under default_path - as configured in BenchMate settings. + Sync and return list of all valid benches under the default_path + configured in BenchMate settings. + + Returns: + list[dict]: List of bench metadata with sites and installed apps. """ settings = get_benchmate_settings() default_path = settings.get("default_path", "/home/karan/benches/") return get_all_benches(default_path) -# ------------------------------------------------------------- +# ? ------------------------------------------------------------- # ? Utility functions -# ------------------------------------------------------------- +# ? ------------------------------------------------------------- def run_cmd(cmd: str, cwd: Path | None = None) -> tuple[str | None, str | None]: """ - Execute a shell command and return stdout as string. - Returns (stdout, None) if success. - Returns (None, "command: error_message") if the command fails. + Execute a shell command. + + Args: + cmd (str): Command to run. + cwd (Path | None): Directory to execute in. + + Returns: + tuple: (stdout, None) if success, else (None, error_message). """ try: out = subprocess.check_output(cmd, cwd=cwd, shell=True, text=True, stderr=subprocess.STDOUT).strip() @@ -40,13 +48,20 @@ def run_cmd(cmd: str, cwd: Path | None = None) -> tuple[str | None, str | None]: def get_git_remote(app_path: Path) -> str | None: """ - Extract repo URL from .git/config (prefer upstream → origin). + Get the Git remote URL for a given app. + Checks upstream first, then origin. + + Args: + app_path (Path): Path to app directory. + + Returns: + str | None: Remote repo URL if found. """ git_config = app_path / ".git" / "config" if not git_config.exists(): return None - config = configparser.ConfigParser(strict=False) # allow duplicate options + config = configparser.ConfigParser(strict=False) # ? allow duplicate keys try: config.read(git_config) if config.has_section('remote "upstream"'): @@ -58,12 +73,18 @@ def get_git_remote(app_path: Path) -> str | None: return None -# ------------------------------------------------------------- +# ? ------------------------------------------------------------- # ? App title extraction -# ------------------------------------------------------------- +# ? ------------------------------------------------------------- def _parse_hooks_title(hooks_path: Path) -> str | None: """ - Parse hooks.py safely using AST to extract app_title. + Extract app_title from hooks.py using AST. + + Args: + hooks_path (Path): Path to hooks.py. + + Returns: + str | None: app_title if found. """ if not hooks_path.exists(): return None @@ -83,7 +104,14 @@ def _parse_hooks_title(hooks_path: Path) -> str | None: def _parse_pyproject_name(pyproject_path: Path) -> str | None: """ - Read project/app name from pyproject.toml ([project] or [tool.poetry]). + Extract project/app name from pyproject.toml. + Supports [project] and [tool.poetry]. + + Args: + pyproject_path (Path): Path to pyproject.toml. + + Returns: + str | None: Name if found. """ if not pyproject_path.exists(): return None @@ -93,6 +121,7 @@ def _parse_pyproject_name(pyproject_path: Path) -> str | None: data = tomllib.loads(pyproject_path.read_text(encoding="utf-8")) except Exception: + # ? Fallback: naive text parsing raw = pyproject_path.read_text(encoding="utf-8") for line in raw.splitlines(): if line.strip().startswith("name"): @@ -115,10 +144,19 @@ def _parse_pyproject_name(pyproject_path: Path) -> str | None: def get_app_title(app_path: Path, app_name: str) -> str: """ - Best-effort app title resolution: - 1. hooks.py (app_title) - 2. pyproject.toml (project.name / poetry.name) - 3. Fallback: prettified app_name + Resolve a human-readable app title. + + Order: + 1. hooks.py → app_title + 2. pyproject.toml → name + 3. fallback: prettified app_name + + Args: + app_path (Path): Path to app folder. + app_name (str): Name of app. + + Returns: + str: App title. """ hooks_title = _parse_hooks_title(app_path / app_name / "hooks.py") if hooks_title: @@ -131,15 +169,26 @@ def get_app_title(app_path: Path, app_name: str) -> str: return app_name.replace("-", " ").replace("_", " ").title() -# ------------------------------------------------------------- +# ? ------------------------------------------------------------- # ? Bench & site parsing -# ------------------------------------------------------------- +# ? ------------------------------------------------------------- def _robust_load_json_array(raw: str): + """ + Try to parse output as JSON/array safely. + Handles garbage before/after JSON. + + Args: + raw (str): Raw CLI output. + + Returns: + Any: Parsed JSON object/list. + """ try: return json.loads(raw) except Exception: pass + # ? Try substring extraction start = raw.find("[") end = raw.rfind("]") if start != -1 and end != -1 and end > start: @@ -149,6 +198,7 @@ def _robust_load_json_array(raw: str): except Exception: pass + # ? Last fallback: literal_eval try: return ast.literal_eval(raw) except Exception: @@ -156,6 +206,15 @@ def _robust_load_json_array(raw: str): def parse_installed_apps(entry: Path) -> tuple[dict, str | None, str | None, str | None]: + """ + Parse installed apps for a bench. + + Args: + entry (Path): Path to bench directory. + + Returns: + tuple: (apps_dict, frappe_version, frappe_branch, error_message) + """ installed_apps = {} frappe_version, frappe_branch, error_message = None, None, None @@ -172,6 +231,7 @@ def parse_installed_apps(entry: Path) -> tuple[dict, str | None, str | None, str frappe.log_error(f"bench version returned non-JSON output for {entry}: {e}", "BenchMate Sync") return installed_apps, None, None, None + # ? Normalize structure try: if isinstance(apps_data, dict): found_list = None @@ -187,6 +247,7 @@ def parse_installed_apps(entry: Path) -> tuple[dict, str | None, str | None, str except Exception: apps = [] + # ? Collect app details try: for app in apps: if not isinstance(app, dict): @@ -219,6 +280,17 @@ def parse_installed_apps(entry: Path) -> tuple[dict, str | None, str | None, str def get_site_apps(bench_path: Path, site_name: str, bench_apps: dict) -> tuple[dict, str | None]: + """ + Get installed apps for a site within a bench. + + Args: + bench_path (Path): Path to bench. + site_name (str): Name of site. + bench_apps (dict): Available bench apps metadata. + + Returns: + tuple: (site_apps, error_message) + """ cmd = f"bench --site {site_name} list-apps --format json" result, err = run_cmd(cmd, cwd=bench_path) @@ -230,7 +302,7 @@ def get_site_apps(bench_path: Path, site_name: str, bench_apps: dict) -> tuple[d site_apps = {} try: - # try robust parser (handles garbage around JSON) + # ? Use robust parser (handles stray output) data = _robust_load_json_array(result) if isinstance(data, dict): @@ -253,6 +325,15 @@ def get_site_apps(bench_path: Path, site_name: str, bench_apps: dict) -> tuple[d def get_all_benches(default_path: str): + """ + Scan a given path and return all valid benches with metadata. + + Args: + default_path (str): Root benches path. + + Returns: + list[dict]: List of benches with their sites, apps, and status. + """ benches: list[dict] = [] root = Path(default_path).expanduser().resolve() @@ -267,6 +348,7 @@ def get_all_benches(default_path: str): sites_path = entry / "sites" procfile_path = entry / "Procfile" + # ? Identify valid bench (must have sites/ + Procfile) if not (sites_path.is_dir() and procfile_path.is_file()): continue From 9a287e0e0d77b439e49bbd756c9f806a25f36e4a Mon Sep 17 00:00:00 2001 From: karan-sanskar Date: Mon, 18 Aug 2025 00:28:49 +0530 Subject: [PATCH 3/5] [Chore]: Key mapping --- benchmate/api/sync.py | 7 ++++--- .../benchmate/doctype/bm_bench/bm_bench.json | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/benchmate/api/sync.py b/benchmate/api/sync.py index 6c1d7ab..22fc8da 100644 --- a/benchmate/api/sync.py +++ b/benchmate/api/sync.py @@ -264,11 +264,12 @@ def parse_installed_apps(entry: Path) -> tuple[dict, str | None, str | None, str app_title = get_app_title(app_path, app_name) installed_apps[app_name] = { - "title": app_title, + "app_name": app_name, + "app_title": app_title, "branch": app_branch, "version": app_version, "commit": app_commit, - "repo": app_repo, + "link": app_repo, } if app_name == "frappe": @@ -376,7 +377,7 @@ def get_all_benches(default_path: str): benches.append( { - "name": entry.name, + "bench_name": entry.name, "path": str(entry), "version": frappe_version, "branch": frappe_branch, diff --git a/benchmate/benchmate/doctype/bm_bench/bm_bench.json b/benchmate/benchmate/doctype/bm_bench/bm_bench.json index 385def9..31dcf0e 100644 --- a/benchmate/benchmate/doctype/bm_bench/bm_bench.json +++ b/benchmate/benchmate/doctype/bm_bench/bm_bench.json @@ -11,12 +11,14 @@ "last_synced_on", "column_break_nigq", "status", - "branch", "version", + "branch", "section_break_gudv", "installed_apps", "section_break_vxsq", - "description" + "description", + "section_break_oxtw", + "error_message" ], "fields": [ { @@ -82,6 +84,16 @@ "in_list_view": 1, "in_standard_filter": 1, "label": "Branch" + }, + { + "fieldname": "section_break_oxtw", + "fieldtype": "Section Break" + }, + { + "depends_on": "eval:doc.status===\"Error\";", + "fieldname": "error_message", + "fieldtype": "Long Text", + "label": "Error Message" } ], "grid_page_length": 50, @@ -93,7 +105,7 @@ "link_fieldname": "bench_name" } ], - "modified": "2025-08-17 21:32:33.248615", + "modified": "2025-08-18 00:21:51.663335", "modified_by": "Administrator", "module": "BenchMate", "name": "BM Bench", From 43fbaadaea1ef1f170bcc0b47875415032bbc052 Mon Sep 17 00:00:00 2001 From: karan-sanskar Date: Mon, 18 Aug 2025 01:53:40 +0530 Subject: [PATCH 4/5] [Chore]: Sync Doctype and Apps Sync Done --- benchmate/api/sync.py | 236 +++++++++++++++++- .../benchmate/doctype/bm_app/bm_app.json | 5 +- .../benchmate/doctype/bm_bench/bm_bench.json | 7 +- .../bm_installed_apps/bm_installed_apps.json | 4 +- .../benchmate/doctype/bm_site/bm_site.json | 5 +- .../benchmate/doctype/bm_sync_log/__init__.py | 0 .../doctype/bm_sync_log/bm_sync_log.js | 8 + .../doctype/bm_sync_log/bm_sync_log.json | 57 +++++ .../doctype/bm_sync_log/bm_sync_log.py | 9 + .../doctype/bm_sync_log/test_bm_sync_log.py | 20 ++ 10 files changed, 339 insertions(+), 12 deletions(-) create mode 100644 benchmate/benchmate/doctype/bm_sync_log/__init__.py create mode 100644 benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.js create mode 100644 benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.json create mode 100644 benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.py create mode 100644 benchmate/benchmate/doctype/bm_sync_log/test_bm_sync_log.py diff --git a/benchmate/api/sync.py b/benchmate/api/sync.py index 22fc8da..30db293 100644 --- a/benchmate/api/sync.py +++ b/benchmate/api/sync.py @@ -9,8 +9,240 @@ from benchmate.api.utils import get_benchmate_settings +# ! benchmate.api.sync.sync_bench_details @frappe.whitelist() -def sync(): +def sync_bench_details(): + """ + Sync all benches found under the default path into the 'BM Bench' DocType. + + Workflow: + - Creates a new Bench record if it does not exist. + - Updates fields if the Bench already exists. + - Marks Bench as "Error" if any error was captured during sync. + - Syncs installed apps into BM App and BM Bench doctypes. + + Returns: + dict: { + "success": bool, + "message": str, + "data": { + "updated_benches": list[str], # names of updated Bench docs + "updated_apps": list[str], # names of updated App docs + } | None + } + """ + try: + # ? Get default benches path from settings + settings = get_benchmate_settings() + default_path = settings.get("default_path", "/home/karan/benches/") + + # ? Gather all benches under the default path + benches = get_all_benches(default_path) + updated_benches, updated_apps = [], [] + + # ? Process each bench and sync into DocType + for bench in benches: + bench_name = bench["bench_name"] + + # ? Create or fetch BM Bench doc + bench_doc = ( + frappe.get_doc("BM Bench", bench_name) + if frappe.db.exists("BM Bench", bench_name) + else frappe.new_doc("BM Bench") + ) + + # ? If error, set status as "Error" and capture details + if bench.get("is_error", False): + bench_doc.update( + { + "status": "Error", + "error_message": bench.get("error_message"), + "last_synced_on": frappe.utils.now(), + "bench_name": bench_name, + "path": bench.get("path"), + "branch": bench.get("branch"), + "version": bench.get("version"), + } + ) + + # ? If no error, update valid bench details + else: + bench_doc.update( + { + "bench_name": bench_name, + "path": bench.get("path"), + "branch": bench.get("branch"), + "version": bench.get("version"), + "error_message": None, + "last_synced_on": frappe.utils.now(), + } + ) + + # ? Manage installed apps + if bench.get("installed_apps"): + # Sync installed apps in BM Apps & BM Bench doctypes + bench_doc, synced_apps = sync_app_details(bench_doc, bench.get("installed_apps")) + + # Track updated apps if any changes were made + if synced_apps: + updated_apps.extend(synced_apps) + + # ? Save or update the Bench document + bench_doc.save(ignore_permissions=True) + updated_benches.append(bench_doc.name) + + # ? Deduplicate lists + updated_benches = list(set(updated_benches)) + updated_apps = list(set(updated_apps)) + + except Exception as e: + # ? Rollback in case of failure + frappe.db.rollback() + + log = json.dumps( + { + "success": False, + "message": f"Bench sync failed: {e!s}", + "data": { + "updated_benches": updated_benches if "updated_benches" in locals() else [], + "updated_apps": updated_apps if "updated_apps" in locals() else [], + }, + }, + indent=4, + ) + + # ? Log sync failure + frappe.get_doc( + { + "doctype": "BM Sync Log", + "title": "Error In Bench Sync", + "log": log, + } + ).insert() + frappe.db.commit() + + return { + "success": False, + "message": f"Bench sync failed: {e!s}", + "data": None, + } + + else: + # ? Prepare sync success log + log = json.dumps( + { + "success": True, + "message": "Benches synced successfully.", + "data": { + "updated_benches": updated_benches, + "updated_apps": updated_apps, + }, + }, + indent=4, + ) + + # ? Log sync success + frappe.get_doc( + { + "doctype": "BM Sync Log", + "title": "Benches Synced Successfully", + "log": log, + } + ).insert() + frappe.db.commit() + + # ? Show UI message + frappe.msgprint( + msg="Benches synced successfully.", + title="Benches Synced", + alert=True, + ) + + return { + "success": True, + "message": "Benches synced successfully.", + "data": { + "updated_benches": updated_benches, + "updated_apps": updated_apps, + }, + } + + +def sync_app_details(bench_doc, installed_apps: dict): + """ + Sync installed apps into the 'BM App' and 'BM Bench' DocTypes. + + Workflow: + - Clears and resets `installed_apps` table in the BM Bench doc. + - Creates a BM App record if it does not exist. + - Updates fields if the App already exists. + - Saves only if there are changes. + + Args: + bench_doc (Document): The BM Bench document being updated. + installed_apps (dict): Dictionary of installed apps with details. + + Returns: + tuple: + - bench_doc (Document): Updated BM Bench document with installed apps. + - updated (list[str]): List of updated BM App names. + """ + updated = [] + + # ? Reset installed_apps table in Bench doc + bench_doc.installed_apps = [] + + # ? Process each installed app + for app in installed_apps.values(): + is_dirty = False + app_name = app.get("app_name") + + # ? Create or fetch BM App doc + app_doc = ( + frappe.get_doc("BM App", app_name) + if frappe.db.exists("BM App", app_name) + else frappe.new_doc("BM App") + ) + + # ? Update app_name if changed + if app_doc.get("app_name") != app_name: + app_doc.update({"app_name": app_name}) + is_dirty = True + + # ? Update app_title if changed + if app_doc.get("app_title") != app.get("app_title"): + app_doc.update({"app_title": app.get("app_title")}) + is_dirty = True + + # ? Update link if changed + if app_doc.get("link") != app.get("link"): + app_doc.update({"link": app.get("link")}) + is_dirty = True + + # ? Save BM App only if changes were made + if is_dirty: + app_doc.save(ignore_permissions=True) + updated.append(app_doc.name) + + # ? Add app details to installed_apps table in Bench doc + bench_doc.append( + "installed_apps", + { + "app_name": app_name, + "app_title": app.get("app_title"), + "branch": app.get("branch"), + "version": app.get("version"), + "link": app.get("link"), + "commit": app.get("commit"), + }, + ) + + return bench_doc, updated + + +# ! benchmate.api.sync.test_sync +@frappe.whitelist() +def test_sync(): """ Sync and return list of all valid benches under the default_path configured in BenchMate settings. @@ -379,8 +611,8 @@ def get_all_benches(default_path: str): { "bench_name": entry.name, "path": str(entry), - "version": frappe_version, "branch": frappe_branch, + "version": frappe_version, "sites": sites, "installed_apps": bench_apps, "is_error": is_error, diff --git a/benchmate/benchmate/doctype/bm_app/bm_app.json b/benchmate/benchmate/doctype/bm_app/bm_app.json index b3efa7b..c8709aa 100644 --- a/benchmate/benchmate/doctype/bm_app/bm_app.json +++ b/benchmate/benchmate/doctype/bm_app/bm_app.json @@ -65,7 +65,7 @@ "table_fieldname": "installed_apps" } ], - "modified": "2025-08-17 20:10:20.850577", + "modified": "2025-08-18 01:17:04.908817", "modified_by": "Administrator", "module": "BenchMate", "name": "BM App", @@ -89,5 +89,6 @@ "sort_field": "creation", "sort_order": "DESC", "states": [], - "title_field": "app_title" + "title_field": "app_title", + "track_changes": 1 } diff --git a/benchmate/benchmate/doctype/bm_bench/bm_bench.json b/benchmate/benchmate/doctype/bm_bench/bm_bench.json index 31dcf0e..3bba017 100644 --- a/benchmate/benchmate/doctype/bm_bench/bm_bench.json +++ b/benchmate/benchmate/doctype/bm_bench/bm_bench.json @@ -11,8 +11,8 @@ "last_synced_on", "column_break_nigq", "status", - "version", "branch", + "version", "section_break_gudv", "installed_apps", "section_break_vxsq", @@ -105,7 +105,7 @@ "link_fieldname": "bench_name" } ], - "modified": "2025-08-18 00:21:51.663335", + "modified": "2025-08-18 01:17:09.426283", "modified_by": "Administrator", "module": "BenchMate", "name": "BM Bench", @@ -145,5 +145,6 @@ "color": "Red", "title": "Error" } - ] + ], + "track_changes": 1 } diff --git a/benchmate/benchmate/doctype/bm_installed_apps/bm_installed_apps.json b/benchmate/benchmate/doctype/bm_installed_apps/bm_installed_apps.json index e29baa7..acf8985 100644 --- a/benchmate/benchmate/doctype/bm_installed_apps/bm_installed_apps.json +++ b/benchmate/benchmate/doctype/bm_installed_apps/bm_installed_apps.json @@ -23,7 +23,6 @@ "options": "BM App" }, { - "fetch_from": "app_name.app_title", "fieldname": "app_title", "fieldtype": "Data", "in_list_view": 1, @@ -42,7 +41,6 @@ "label": "Version" }, { - "fetch_from": "app_name.link", "fieldname": "link", "fieldtype": "Data", "in_list_view": 1, @@ -63,7 +61,7 @@ "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2025-08-17 21:44:59.925833", + "modified": "2025-08-18 01:17:56.627642", "modified_by": "Administrator", "module": "BenchMate", "name": "BM Installed Apps", diff --git a/benchmate/benchmate/doctype/bm_site/bm_site.json b/benchmate/benchmate/doctype/bm_site/bm_site.json index 6902151..f1f9287 100644 --- a/benchmate/benchmate/doctype/bm_site/bm_site.json +++ b/benchmate/benchmate/doctype/bm_site/bm_site.json @@ -79,7 +79,7 @@ "grid_page_length": 50, "index_web_pages_for_search": 1, "links": [], - "modified": "2025-08-17 20:11:26.287364", + "modified": "2025-08-18 01:17:13.172040", "modified_by": "Administrator", "module": "BenchMate", "name": "BM Site", @@ -120,5 +120,6 @@ "title": "Error" } ], - "title_field": "site_name" + "title_field": "site_name", + "track_changes": 1 } diff --git a/benchmate/benchmate/doctype/bm_sync_log/__init__.py b/benchmate/benchmate/doctype/bm_sync_log/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.js b/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.js new file mode 100644 index 0000000..6262056 --- /dev/null +++ b/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.js @@ -0,0 +1,8 @@ +// Copyright (c) 2025, Karan Mistry and contributors +// For license information, please see license.txt + +// frappe.ui.form.on("BM Sync Log", { +// refresh(frm) { + +// }, +// }); diff --git a/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.json b/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.json new file mode 100644 index 0000000..975fd64 --- /dev/null +++ b/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.json @@ -0,0 +1,57 @@ +{ + "actions": [], + "allow_rename": 1, + "autoname": "format:BM-SL-{#}", + "creation": "2025-08-18 01:20:36.254906", + "doctype": "DocType", + "engine": "MyISAM", + "field_order": [ + "title", + "log" + ], + "fields": [ + { + "fieldname": "title", + "fieldtype": "Data", + "in_list_view": 1, + "label": "Title", + "read_only": 1 + }, + { + "fieldname": "log", + "fieldtype": "Long Text", + "in_list_view": 1, + "label": "Log", + "read_only": 1 + } + ], + "grid_page_length": 50, + "index_web_pages_for_search": 1, + "links": [], + "modified": "2025-08-18 01:22:33.862949", + "modified_by": "Administrator", + "module": "BenchMate", + "name": "BM Sync Log", + "naming_rule": "Expression", + "owner": "Administrator", + "permissions": [ + { + "create": 1, + "delete": 1, + "email": 1, + "export": 1, + "print": 1, + "read": 1, + "report": 1, + "role": "System Manager", + "share": 1, + "write": 1 + } + ], + "row_format": "Dynamic", + "sort_field": "creation", + "sort_order": "DESC", + "states": [], + "title_field": "title", + "track_seen": 1 +} diff --git a/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.py b/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.py new file mode 100644 index 0000000..7d0f3ed --- /dev/null +++ b/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.py @@ -0,0 +1,9 @@ +# Copyright (c) 2025, Karan Mistry and contributors +# For license information, please see license.txt + +# import frappe +from frappe.model.document import Document + + +class BMSyncLog(Document): + pass diff --git a/benchmate/benchmate/doctype/bm_sync_log/test_bm_sync_log.py b/benchmate/benchmate/doctype/bm_sync_log/test_bm_sync_log.py new file mode 100644 index 0000000..3e49b0e --- /dev/null +++ b/benchmate/benchmate/doctype/bm_sync_log/test_bm_sync_log.py @@ -0,0 +1,20 @@ +# Copyright (c) 2025, Karan Mistry and Contributors +# See license.txt + +# import frappe +from frappe.tests import IntegrationTestCase + +# On IntegrationTestCase, the doctype test records and all +# link-field test record dependencies are recursively loaded +# Use these module variables to add/remove to/from that list +EXTRA_TEST_RECORD_DEPENDENCIES = [] # eg. ["User"] +IGNORE_TEST_RECORD_DEPENDENCIES = [] # eg. ["User"] + + +class IntegrationTestBMSyncLog(IntegrationTestCase): + """ + Integration tests for BMSyncLog. + Use this class for testing interactions between multiple components. + """ + + pass From f3be1286bcb758ef06f536475938d80d085da9af Mon Sep 17 00:00:00 2001 From: karan-sanskar Date: Mon, 18 Aug 2025 03:26:59 +0530 Subject: [PATCH 5/5] [Feat]: Set button to sync bench, auto set cwd in settiings and etc --- benchmate/api/sync.py | 254 +++++++++++++----- .../doctype/bm_settings/bm_settings.js | 45 +++- .../doctype/bm_sync_log/bm_sync_log.json | 37 ++- benchmate/hooks.py | 2 +- 4 files changed, 271 insertions(+), 67 deletions(-) diff --git a/benchmate/api/sync.py b/benchmate/api/sync.py index 30db293..18fde8e 100644 --- a/benchmate/api/sync.py +++ b/benchmate/api/sync.py @@ -1,6 +1,7 @@ import ast import configparser import json +import os import subprocess from pathlib import Path @@ -9,6 +10,37 @@ from benchmate.api.utils import get_benchmate_settings +# ! benchmate.api.sync.enqueue_sync_bench_details +@frappe.whitelist() +def enqueue_sync_bench_details(): + """ + Enqueue the `sync_bench_details` function to run asynchronously using Frappe's background jobs. + + Queue: default + Timeout: 600 seconds + + Usage: + Called to process bench syncing in the background without blocking the main thread. + """ + try: + # ? Enqueue the sync_bench_details task in Frappe background queue + frappe.enqueue(sync_bench_details, queue="default", timeout=600) + except Exception as e: + # ? If enqueue fails, return error status + return { + "success": False, + "message": f"Failed to enqueue bench sync: {e!s}", + "data": None, + } + else: + # ? On success, return success message + return { + "success": True, + "message": "Bench sync task enqueued successfully.", + "data": {"queued_function": "sync_bench_details"}, + } + + # ! benchmate.api.sync.sync_bench_details @frappe.whitelist() def sync_bench_details(): @@ -20,16 +52,18 @@ def sync_bench_details(): - Updates fields if the Bench already exists. - Marks Bench as "Error" if any error was captured during sync. - Syncs installed apps into BM App and BM Bench doctypes. + - Syncs sites into BM Site doctype. Returns: - dict: { - "success": bool, - "message": str, - "data": { - "updated_benches": list[str], # names of updated Bench docs - "updated_apps": list[str], # names of updated App docs - } | None - } + dict: { + "success": bool, + "message": str, + "data": { + "updated_benches": list[str], # names of updated Bench docs + "updated_apps": list[str], # names of updated App docs + "updated_sites": list[str], # names of updated Site docs + } | None + } """ try: # ? Get default benches path from settings @@ -38,7 +72,7 @@ def sync_bench_details(): # ? Gather all benches under the default path benches = get_all_benches(default_path) - updated_benches, updated_apps = [], [] + updated_benches, updated_apps, updated_sites = [], [], [] # ? Process each bench and sync into DocType for bench in benches: @@ -80,45 +114,54 @@ def sync_bench_details(): # ? Manage installed apps if bench.get("installed_apps"): - # Sync installed apps in BM Apps & BM Bench doctypes + # ? Sync installed apps in BM Apps & BM Bench doctypes bench_doc, synced_apps = sync_app_details(bench_doc, bench.get("installed_apps")) - # Track updated apps if any changes were made + # ? Track updated apps if any changes were made if synced_apps: updated_apps.extend(synced_apps) # ? Save or update the Bench document bench_doc.save(ignore_permissions=True) - updated_benches.append(bench_doc.name) + updated_benches.append(bench_doc.get("name")) + + # ? Manage sites of the bench + if bench.get("sites"): + synced_sites = sync_site_details(bench_doc, bench.get("sites")) + + # ? Track updated sites if any changes were made + if synced_sites: + updated_sites.extend(synced_sites) # ? Deduplicate lists updated_benches = list(set(updated_benches)) updated_apps = list(set(updated_apps)) + updated_sites = list(set(updated_sites)) except Exception as e: # ? Rollback in case of failure frappe.db.rollback() - log = json.dumps( - { - "success": False, - "message": f"Bench sync failed: {e!s}", - "data": { - "updated_benches": updated_benches if "updated_benches" in locals() else [], - "updated_apps": updated_apps if "updated_apps" in locals() else [], - }, - }, - indent=4, - ) + # ? Prepare error log + message = "Error In Bench Sync" + data = { + "updated_benches": updated_benches if "updated_benches" in locals() else [], + "updated_apps": updated_apps if "updated_apps" in locals() else [], + "updated_sites": updated_sites if "updated_sites" in locals() else [], + } - # ? Log sync failure + # ? Log sync error frappe.get_doc( { "doctype": "BM Sync Log", - "title": "Error In Bench Sync", - "log": log, + "title": message, + "status": "Error", + "log": json.dumps(data, indent=4), } ).insert() + frappe.log_error(message, str(e)) + + # ? Commit the BM Sync Log frappe.db.commit() return { @@ -128,43 +171,34 @@ def sync_bench_details(): } else: - # ? Prepare sync success log - log = json.dumps( - { - "success": True, - "message": "Benches synced successfully.", - "data": { - "updated_benches": updated_benches, - "updated_apps": updated_apps, - }, - }, - indent=4, - ) + # ? Prepare success log + message = "Benches synced successfully." + data = { + "updated_benches": updated_benches, + "updated_apps": updated_apps, + "updated_sites": updated_sites, + } # ? Log sync success frappe.get_doc( { "doctype": "BM Sync Log", - "title": "Benches Synced Successfully", - "log": log, + "title": message, + "status": "Success", + "log": json.dumps(data, indent=4), } ).insert() + + # ? Commit the changes if success frappe.db.commit() # ? Show UI message - frappe.msgprint( - msg="Benches synced successfully.", - title="Benches Synced", - alert=True, - ) + frappe.msgprint(msg=message, alert=True) return { "success": True, - "message": "Benches synced successfully.", - "data": { - "updated_benches": updated_benches, - "updated_apps": updated_apps, - }, + "message": message, + "data": data, } @@ -222,7 +256,7 @@ def sync_app_details(bench_doc, installed_apps: dict): # ? Save BM App only if changes were made if is_dirty: app_doc.save(ignore_permissions=True) - updated.append(app_doc.name) + updated.append(app_doc.get("name")) # ? Add app details to installed_apps table in Bench doc bench_doc.append( @@ -240,6 +274,68 @@ def sync_app_details(bench_doc, installed_apps: dict): return bench_doc, updated +def sync_site_details(bench_doc, sites): + """ + Sync site details for a given bench into BM Site doctype. + + Args: + bench_doc (dict): Bench document details containing bench_name, status, etc. + sites (dict): Dictionary of sites under this bench with keys like site_name, path, installed_apps. + + Returns: + list: List of BM Site document names that were created/updated. + """ + updated = [] + + for site in sites.values(): + # ? Generate unique BM Site name using bench + site_name + site_name_key = f"{bench_doc.get('bench_name')}-{site.get('site_name')}" + + # ? Fetch existing BM Site doc or create a new one if it doesn't exist + site_doc = ( + frappe.get_doc("BM Site", site_name_key) + if frappe.db.exists("BM Site", site_name_key) + else frappe.new_doc("BM Site") + ) + + # ? Update core site details + site_doc.update( + { + "site_name": site.get("site_name"), + "bench_name": bench_doc.get("bench_name"), + "status": bench_doc.get("status"), + "path": site.get("path"), + "last_synced_on": frappe.utils.now(), + } + ) + + # ? Reset installed_apps child table before inserting fresh entries + site_doc.installed_apps = [] + + # ? Loop through installed apps for this site and append them to child table + for app in site.get("installed_apps").values(): + site_doc.append( + "installed_apps", + { + "app_name": app.get("app_name"), + "app_title": app.get("app_title"), + "branch": app.get("branch"), + "version": app.get("version"), + "link": app.get("link"), + "commit": app.get("commit"), + }, + ) + + # ? Save or update BM Site document + site_doc.save() + + # ? Track updated/created site record name + updated.append(site_doc.get("name")) + + # ? Return list of updated site document names + return updated + + # ! benchmate.api.sync.test_sync @frappe.whitelist() def test_sync(): @@ -592,20 +688,18 @@ def get_all_benches(default_path: str): if err: is_error, error_message = True, err - sites = [] + sites = {} for s in sites_path.iterdir(): if s.is_dir() and s.name != "assets": site_apps, site_err = get_site_apps(entry, s.name, bench_apps) if site_err and not error_message: is_error, error_message = True, site_err - sites.append( - { - "site_name": s.name, - "bench_name": entry.name, - "path": str(s), - "installed_apps": site_apps, - } - ) + sites[s.name] = { + "site_name": s.name, + "bench_name": entry.name, + "path": str(s), + "installed_apps": site_apps, + } benches.append( { @@ -621,3 +715,43 @@ def get_all_benches(default_path: str): ) return benches + + +# ! benchmate.api.sync.after_install +@frappe.whitelist() +def after_install(): + """ + Runs after app installation. + Sets the default bench path in BM Settings. + """ + # ? Set the default bench path in BM Settings after installation + set_default_bench_path() + + +def set_default_bench_path(): + """ + Updates the 'default_path' field in BM Settings with the current bench directory. + """ + # ? Get the current bench directory + default_bench_path = get_current_bench_dir() + + # ? Update the BM Settings DocType with the default path + frappe.db.set_value("BM Settings", "BM Settings", "default_path", default_bench_path) + + +def get_current_bench_dir(): + """ + Returns the current bench directory. + + Workflow: + - Fetches the path to the 'frappe' app + - Moves up directories to get the bench root + """ + # ? Get the path to the 'frappe' app within the current bench + frappe_app_path = frappe.get_app_source_path("frappe") + + # ? Move up three directories to reach the root bench folder + current_bench_path = os.path.abspath(os.path.join(frappe_app_path, "../../../")) + + # ? Return the bench directory path + return current_bench_path diff --git a/benchmate/benchmate/doctype/bm_settings/bm_settings.js b/benchmate/benchmate/doctype/bm_settings/bm_settings.js index d2b04de..f6f0726 100644 --- a/benchmate/benchmate/doctype/bm_settings/bm_settings.js +++ b/benchmate/benchmate/doctype/bm_settings/bm_settings.js @@ -1,8 +1,45 @@ // Copyright (c) 2025, Karan Mistry and contributors // For license information, please see license.txt -// frappe.ui.form.on("BM Settings", { -// refresh(frm) { +frappe.ui.form.on("BM Settings", { + refresh(frm) { + // ? Add the "Sync Bench Details" button when the form is refreshed + addSyncBenchDetailsButton(frm); + }, +}); -// }, -// }); +function addSyncBenchDetailsButton(frm) { + // ? Check if the BM Settings document has syncing enabled + if (frm.doc.enable) { + // ? Add custom button to trigger bench details sync + frm.add_custom_button("Sync Bench Details", () => { + enqueueSyncBenchDetailsButton(); + }); + } +} + +function enqueueSyncBenchDetailsButton() { + // ? Call Frappe backend method to enqueue sync + frappe.call({ + method: "benchmate.api.sync.enqueue_sync_bench_details", + args: {}, + + // ? Freeze the UI with message while processing + freeze: true, + freeze_message: "Enqueuing Sync Bench Details...", + + // ? Callback when backend call is complete + callback: function (res) { + // ? If enqueue was successful, show a success alert + if (res.message.success) { + frappe.show_alert( + { + message: __("Successfully Enqueued Sync Bench Details!"), + indicator: "green", + }, + 5 + ); + } + }, + }); +} diff --git a/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.json b/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.json index 975fd64..7380294 100644 --- a/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.json +++ b/benchmate/benchmate/doctype/bm_sync_log/bm_sync_log.json @@ -7,6 +7,9 @@ "engine": "MyISAM", "field_order": [ "title", + "column_break_nerx", + "status", + "section_break_djza", "log" ], "fields": [ @@ -23,12 +26,29 @@ "in_list_view": 1, "label": "Log", "read_only": 1 + }, + { + "default": "Log", + "fieldname": "status", + "fieldtype": "Select", + "in_list_view": 1, + "label": "Status", + "options": "\nError\nSuccess\nLog", + "read_only": 1 + }, + { + "fieldname": "column_break_nerx", + "fieldtype": "Column Break" + }, + { + "fieldname": "section_break_djza", + "fieldtype": "Section Break" } ], "grid_page_length": 50, "index_web_pages_for_search": 1, "links": [], - "modified": "2025-08-18 01:22:33.862949", + "modified": "2025-08-18 02:47:10.030458", "modified_by": "Administrator", "module": "BenchMate", "name": "BM Sync Log", @@ -51,7 +71,20 @@ "row_format": "Dynamic", "sort_field": "creation", "sort_order": "DESC", - "states": [], + "states": [ + { + "color": "Red", + "title": "Error" + }, + { + "color": "Green", + "title": "Success" + }, + { + "color": "Blue", + "title": "Log" + } + ], "title_field": "title", "track_seen": 1 } diff --git a/benchmate/hooks.py b/benchmate/hooks.py index 3b6307e..0bfeba4 100644 --- a/benchmate/hooks.py +++ b/benchmate/hooks.py @@ -86,7 +86,7 @@ # ------------ # before_install = "benchmate.install.before_install" -# after_install = "benchmate.install.after_install" +after_install = "benchmate.api.sync.after_install" # Uninstallation # ------------