From 24e3f1303597e4f81c75b4410b052eff575e8255 Mon Sep 17 00:00:00 2001 From: amangalampalli-ks Date: Thu, 23 Jul 2026 18:34:00 +0530 Subject: [PATCH] Return enterprise user IDs in verbose mode and add tree --format=json with share permissions (#2240) * fix: Return IDs for node/teams/roles in enterprise-info --users -v * Implement format json, nsf folder and all records share-perms for tree * Fix share-permissions for classic share-folder --------- Co-authored-by: sshrushanth-ks --- keepercommander/commands/enterprise.py | 21 +- keepercommander/commands/folder.py | 944 ++++++++++++++++-- .../nested_share_folder/__init__.py | 5 + .../nested_share_folder/acl_cache.py | 230 +++++ .../nested_share_folder/record_api.py | 59 +- keepercommander/nested_share_folder/sync.py | 8 + keepercommander/params.py | 5 + .../service/util/parse_keeper_response.py | 61 +- unit-tests/service/test_response_parser.py | 49 +- unit-tests/test_command_enterprise.py | 23 + unit-tests/test_nsf_acl_cache.py | 473 +++++++++ 11 files changed, 1732 insertions(+), 146 deletions(-) create mode 100644 keepercommander/nested_share_folder/acl_cache.py create mode 100644 unit-tests/test_nsf_acl_cache.py diff --git a/keepercommander/commands/enterprise.py b/keepercommander/commands/enterprise.py index 512c4fce1..c1f592b9e 100644 --- a/keepercommander/commands/enterprise.py +++ b/keepercommander/commands/enterprise.py @@ -781,6 +781,7 @@ def tree_node(node): user_teams[enterprise_user_id].add(team_uid) displayed_columns = [x for x in supported_columns if x in columns] + is_verbose = kwargs.get('verbose') or False rows = [] for u in users.values(): user_status_dict = get_user_status_dict(u) @@ -796,12 +797,19 @@ def tree_node(node): elif column == 'transfer_status': row.append(user_status_dict['acct_transfer_status']) elif column == 'node': - row.append(self.get_node_path(params, u['node_id'])) + if is_verbose: + row.append(str(u['node_id'])) + else: + row.append(self.get_node_path(params, u['node_id'])) elif column == 'team_count': row.append(len([1 for t in teams.values() if t['users'] and user_id in t['users']])) elif column == 'teams': - team_names = [t["name"] for t in teams.values() if t['users'] and user_id in t['users']] - row.append(team_names) + user_team_list = [t for t in teams.values() + if t['users'] and user_id in t['users']] + if is_verbose: + row.append([t['id'] for t in user_team_list]) + else: + row.append([t['name'] for t in user_team_list]) elif column == 'role_count' or column == 'roles': role_ids = set() if user_id in user_roles: @@ -813,8 +821,11 @@ def tree_node(node): if column == 'role_count': row.append(len(role_ids)) else: - role_names = [roles[role_id]['name'] for role_id in role_ids if role_id in roles] - row.append(role_names) + if is_verbose: + row.append([str(role_id) for role_id in role_ids if role_id in roles]) + else: + role_names = [roles[role_id]['name'] for role_id in role_ids if role_id in roles] + row.append(role_names) elif column == 'alias': row.append([x['username'] for x in params.enterprise.get('user_aliases', []) if x['enterprise_user_id'] == user_id and x['username'] != email]) diff --git a/keepercommander/commands/folder.py b/keepercommander/commands/folder.py index 84a1ef7d5..180ea64bb 100644 --- a/keepercommander/commands/folder.py +++ b/keepercommander/commands/folder.py @@ -81,12 +81,16 @@ def register_command_info(aliases, command_info): cd_parser.exit = suppress_exit -tree_parser = argparse.ArgumentParser(prog='tree', description='Display the folder structure.') +tree_parser = argparse.ArgumentParser(prog='tree', description='Display the folder structure.', + parents=[base.json_output_parser]) tree_parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='print ids') -tree_parser.add_argument('-r', '--records', action='store_true', help='show records within each folder') -show_shares_help = 'show share permissions info (shown in parentheses) for each shared folder' -tree_parser.add_argument('-s', '--shares', action='store_true', help=show_shares_help) -perms_key_help = 'hide share permissions key (valid only when used with --shares flag, which shows key by default)' +tree_parser.add_argument('-r', '--records', action='store_true', + help='show records within each folder (includes record type)') +tree_parser.add_argument('-s', '--shares', action='store_true', + help='show classic shared-folder permissions; with -r also classic record shares') +tree_parser.add_argument('-ns', '--nsf-shares', dest='nsf_shares', action='store_true', + help='show NSF folder permissions (ACL API); with -r also NSF record shares') +perms_key_help = 'hide share permissions key (valid with --shares / --nsf-shares)' tree_parser.add_argument('-hk', '--hide-shares-key', action='store_true', help=perms_key_help) tree_parser.add_argument('-t', '--title', action='store', help='show optional title for folder structure') tree_parser.add_argument('folder', nargs='?', type=str, action='store', help='folder path or UID') @@ -422,18 +426,40 @@ def execute(self, params, **kwargs): verbose = kwargs.get('verbose', False) records = kwargs.get('records') shares = kwargs.get('shares') - hide_key = kwargs.get('hide_shares_key', not shares) + nsf_shares = kwargs.get('nsf_shares') + fmt = kwargs.get('format') or 'table' + show_key = bool(shares or nsf_shares) + hide_key = kwargs.get('hide_shares_key', not show_key) title = kwargs.get('title') + trees = [] if folder_name in params.folder_cache: folder = params.folder_cache.get(folder_name) - formatted_tree(params, folder, verbose=verbose, show_records=records, shares=shares, hide_shares_key=hide_key, title=title) + trees.append(formatted_tree( + params, folder, verbose=verbose, show_records=records, shares=shares, + nsf_shares=nsf_shares, hide_shares_key=hide_key, title=title, fmt=fmt)) else: folders, pattern = try_resolve_path(params, folder_name, find_all_matches=True) if not pattern: for idx, folder in enumerate(folders): - formatted_tree(params, folder, verbose=verbose, show_records=records, shares=shares, hide_shares_key=hide_key or idx > 0, title=title) + trees.append(formatted_tree( + params, folder, verbose=verbose, show_records=records, shares=shares, + nsf_shares=nsf_shares, hide_shares_key=hide_key or idx > 0, title=title, + fmt=fmt)) else: raise CommandError('tree', f'Folder {folder_name} not found') + if fmt == 'json': + payload = trees[0] if len(trees) == 1 else {'trees': [t for t in trees if t]} + text = _tree_json_dumps(payload) + output = kwargs.get('output') + if output: + _, ext = os.path.splitext(output) + path = output if ext else output + '.json' + with open(path, 'w', encoding='utf-8') as fd: + fd.write(text) + fd.write('\n') + logging.info('Report path: %s', os.path.abspath(path)) + return None + return text class FolderRenameCommand(Command): @@ -1779,65 +1805,727 @@ def add_subfolders(folder): # type: (BaseFolderNode) -> None api.sync_down(params) -def formatted_tree(params, folder, verbose=False, show_records=False, shares=False, hide_shares_key=False, title=None): - def print_share_permissions_key(): - perms_key = 'Share Permissions Key:\n' \ - '======================\n' \ - 'RO = Read-Only\n' \ - 'MU = Can Manage Users\n' \ - 'MR = Can Manage Records\n' \ - 'CE = Can Edit\n' \ - 'CS = Can Share\n' \ - '======================\n' - print(perms_key) - - def get_share_info(node): - MU_KEY = 'manage_users' - MR_KEY = 'manage_records' - DMR_KEY = 'default_manage_records' - DMU_KEY = 'default_manage_user' - DCE_KEY = 'default_can_edit' - DCS_KEY = 'default_can_share' - perm_abbrev_lookup = {MU_KEY: 'MU', MR_KEY: 'MR', DMR_KEY: 'MU', DMU_KEY: 'MU', DCE_KEY: 'CE', DCS_KEY: 'CS'} - - def get_users_info(users): - info = [] - for u in users: - email = u.get('username') - if email == params.user: +def _resolve_tree_team_name(params, team_uid): + if not team_uid: + return '' + team = (getattr(params, 'team_cache', None) or {}).get(team_uid) or {} + name = team.get('name') if isinstance(team, dict) else getattr(team, 'name', None) + if name: + return name + if params.enterprise: + for t in params.enterprise.get('teams') or []: + if t.get('team_uid') == team_uid: + name = t.get('name') + if name: + return name + return team_uid + + +def _resolve_sf_member_email(params, user_entry): + """Resolve a shared-folder user row to an email (never a raw account UID).""" + email = (user_entry.get('username') or '').strip() + if email and '@' in email: + return email + account_uid = user_entry.get('account_uid') or '' + if account_uid: + cached = (getattr(params, 'user_cache', None) or {}).get(account_uid) + if cached and '@' in str(cached): + return str(cached) + if params.enterprise: + for u in params.enterprise.get('users') or []: + if u.get('user_account_uid') == account_uid and u.get('username'): + return u.get('username') + return email if email and '@' in email else '' + + +def _is_compact_share_entry(obj): + """True for leaf share objects like {email, permissions} / {name, uid, permissions}.""" + if not isinstance(obj, dict) or not obj: + return False + allowed = {'email', 'name', 'uid', 'permissions'} + if set(obj.keys()) - allowed: + return False + for k, v in obj.items(): + if k == 'permissions': + if not isinstance(v, list) or not all(isinstance(x, str) for x in v): + return False + elif not isinstance(v, str): + return False + return True + + +def _dump_compact_share_entry(obj): + """Dump a share entry on one line; keep field order name/email, uid, permissions.""" + parts = [] + for key in ('name', 'email', 'uid', 'permissions'): + if key not in obj: + continue + val = obj[key] + if key == 'permissions': + inner = ', '.join(json.dumps(x) for x in val) + parts.append(f'"{key}": [{inner}]') + else: + parts.append(f'"{key}": {json.dumps(val)}') + return '{ ' + ', '.join(parts) + ' }' + + +def _tree_json_dumps(obj, level=0, indent=2): + """Pretty JSON with share entries / permission arrays kept on one line.""" + sp = ' ' * (indent * level) + sp1 = ' ' * (indent * (level + 1)) + if isinstance(obj, dict): + if not obj: + return '{}' + if _is_compact_share_entry(obj): + return _dump_compact_share_entry(obj) + lines = ['{'] + items = list(obj.items()) + for i, (k, v) in enumerate(items): + comma = ',' if i < len(items) - 1 else '' + dumped = _tree_json_dumps(v, level + 1, indent) + lines.append(f'{sp1}{json.dumps(k)}: {dumped}{comma}') + lines.append(sp + '}') + return '\n'.join(lines) + if isinstance(obj, list): + if not obj: + return '[]' + if all(isinstance(x, str) for x in obj): + return '[' + ', '.join(json.dumps(x) for x in obj) + ']' + if all(_is_compact_share_entry(x) for x in obj): + lines = ['['] + for i, x in enumerate(obj): + comma = ',' if i < len(obj) - 1 else '' + lines.append(f'{sp1}{_dump_compact_share_entry(x)}{comma}') + lines.append(sp + ']') + return '\n'.join(lines) + lines = ['['] + for i, x in enumerate(obj): + comma = ',' if i < len(obj) - 1 else '' + dumped = _tree_json_dumps(x, level + 1, indent) + lines.append(f'{sp1}{dumped}{comma}') + lines.append(sp + ']') + return '\n'.join(lines) + return json.dumps(obj) + + +_NSF_ROLE_ABBREV = { + 'viewer': 'VW', + 'contributor': 'CT', + 'share-manager': 'SM', + 'content-manager': 'CM', + 'content-share-manager': 'CSM', + 'full-manager': 'FM', + 'unresolved': 'UN', + 'owner': 'OW', +} + + +def _nsf_role_label(accessor): + from .nested_share_folder.helpers import format_role_display, get_access_role_label + role = accessor.get('role') + if role: + return format_role_display(role) + return get_access_role_label(accessor) or 'viewer' + + +def _nsf_role_abbrev(accessor): + label = _nsf_role_label(accessor) + return _NSF_ROLE_ABBREV.get(label, (label[:2].upper() if label else 'VW')) + + +def _looks_like_uid(value): + if not value or not isinstance(value, str) or '@' in value: + return False + # Keeper UIDs are typically 22-char base64url + return bool(re.fullmatch(r'[A-Za-z0-9_-]{16,28}', value)) + + +def _resolve_nsf_user_email(params, accessor): + """Resolve AT_USER accessor to an email; never return a raw UID as email.""" + email = (accessor.get('username') or '').strip() + if email and not _looks_like_uid(email): + return email + auid = accessor.get('accessor_uid') or '' + if not auid: + return email if email and not _looks_like_uid(email) else '' + cached = getattr(params, 'user_cache', {}).get(auid) if hasattr(params, 'user_cache') else None + if cached: + return cached + if params.enterprise: + for u in params.enterprise.get('users') or []: + if u.get('user_account_uid') == auid and u.get('username'): + return u.get('username') + try: + from ..nested_share_folder.folder_api import _resolve_uid_to_username + resolved = _resolve_uid_to_username(params, auid) + if resolved: + if not hasattr(params, 'user_cache') or params.user_cache is None: + params.user_cache = {} + params.user_cache[auid] = resolved + return resolved + except Exception as exc: + logging.debug('NSF user resolve failed for %s: %s', auid, exc) + return '' + + +def _resolve_nsf_app_name(params, app_uid): + if not app_uid: + return '' + try: + from .ksm import KSMCommand + rec = KSMCommand.get_app_record(params, app_uid) + if rec: + data = rec.get('data_unencrypted') + if data: + if isinstance(data, (bytes, bytearray)): + data = data.decode('utf-8') + title = json.loads(data).get('title') + if title: + return title + except Exception as exc: + logging.debug('NSF app resolve via KSM failed for %s: %s', app_uid, exc) + if app_uid in (params.record_cache or {}): + try: + r = api.get_record(params, app_uid) + if r and getattr(r, 'title', None): + return r.title + except Exception as exc: + logging.debug('NSF app resolve via record cache failed for %s: %s', app_uid, exc) + return '' + + +def _nsf_folder_share_data(params, folder_uid, *, include_uids=False): + """Return structured NSF folder share perms and a compact text suffix. + + Owner is listed under ``users`` with permission ``OW`` (not a separate label). + Applications are listed separately (never as fake user emails / UIDs). + """ + from .. import nested_share_folder as _nsf + accessors = _nsf.get_nsf_folder_share_accessors(params, folder_uid) + folder_info = (getattr(params, 'nested_share_folders', {}) or {}).get(folder_uid) or {} + owner = (folder_info.get('owner_username') or '').strip() + if owner and _looks_like_uid(owner): + owner = '' + users = [] + teams = [] + applications = [] + user_parts = [] + team_parts = [] + app_parts = [] + seen_users = set() + + if owner: + users.append({'email': owner, 'permissions': ['OW']}) + user_parts.append(f'[{owner}:OW]') + seen_users.add(owner.lower()) + + for a in accessors: + at = a.get('access_type') or '' + abbrev = _nsf_role_abbrev(a) + auid = a.get('accessor_uid') or '' + + if at == 'AT_OWNER': + if not owner: + email = _resolve_nsf_user_email(params, a) + if email and email.lower() not in seen_users: + users.append({'email': email, 'permissions': ['OW']}) + user_parts.append(f'[{email}:OW]') + seen_users.add(email.lower()) + continue + + if at == 'AT_TEAM': + name = _resolve_tree_team_name(params, auid) + entry = {'name': name, 'permissions': [abbrev]} + if include_uids and auid: + entry['uid'] = auid + teams.append(entry) + team_parts.append(f'[{name}:{abbrev}]') + continue + + if at == 'AT_APPLICATION': + app_name = _resolve_nsf_app_name(params, auid) or (auid if include_uids else 'application') + entry = {'name': app_name, 'permissions': [abbrev]} + if include_uids and auid: + entry['uid'] = auid + applications.append(entry) + app_parts.append(f'[{app_name}:{abbrev}]') + continue + + if at in ('AT_USER', 'AT_UNKNOWN', ''): + email = _resolve_nsf_user_email(params, a) + if not email: + # Unresolved user: only expose UID when -v, never as email. + if include_uids and auid: + entry = {'uid': auid, 'permissions': [abbrev]} + users.append(entry) + user_parts.append(f'[{auid}:{abbrev}]') + continue + if email.lower() in seen_users or email == params.user: + continue + users.append({'email': email, 'permissions': [abbrev]}) + user_parts.append(f'[{email}:{abbrev}]') + seen_users.add(email.lower()) + + data = {} + if users: + data['users'] = users + if teams: + data['teams'] = teams + if applications: + data['applications'] = applications + if not data: + state = (getattr(params, 'nested_share_folder_sharing_states', {}) or {}).get(folder_uid) or {} + if state.get('shared') or state.get('count', 0) > 0: + data['shared'] = True + data['count'] = state.get('count', 0) + + parts = [] + if user_parts: + parts.append('users:' + ','.join(user_parts)) + if team_parts: + parts.append('teams:' + ','.join(team_parts)) + if app_parts: + parts.append('applications:' + ','.join(app_parts)) + if not parts and data.get('shared'): + parts.append(f'shared:count={data.get("count", 0)}') + text = f' ({"; ".join(parts)})' if parts else '' + return data or None, text + + +def _classic_folder_share_data(params, sf, *, include_uids=False): + """Return structured classic SF share perms and compact text suffix. + + Tree text uses a single ``default:`` blob (all default flags). + JSON splits folder-user defaults vs default record rights: + - user_permissions: MU / MR (default manage users / manage records) + - record_permissions: CE / CS (default can-edit / can-share on records) + Named people/teams remain under ``users`` / ``teams``. + """ + DEFAULT_USER_PERM_KEYS = { + 'default_manage_users': 'MU', + 'default_manage_user': 'MU', # legacy alias if present + 'default_manage_records': 'MR', + } + RECORD_PERM_KEYS = { + 'default_can_edit': 'CE', + 'default_can_share': 'CS', + } + MEMBER_PERM_KEYS = { + 'manage_users': 'MU', + 'manage_records': 'MR', + } + + sf = sf or {} + user_permissions = [abbr for key, abbr in DEFAULT_USER_PERM_KEYS.items() if sf.get(key)] + user_permissions = [a for a in ('MU', 'MR') if a in user_permissions] + record_permissions = [abbr for key, abbr in RECORD_PERM_KEYS.items() if sf.get(key)] + record_permissions = [a for a in ('CE', 'CS') if a in record_permissions] + + default_perms = user_permissions + record_permissions + if not default_perms: + default_perms = ['RO'] + + users = [] + user_parts = [] + seen_emails = set() + for u in sf.get('users') or []: + email = _resolve_sf_member_email(params, u) + account_uid = u.get('account_uid') or '' + privs = [abbr for key_name, abbr in MEMBER_PERM_KEYS.items() if u.get(key_name)] + privs = [a for a in ('MU', 'MR') if a in privs] or ['RO'] + if not email: + # Sync often has account_uid only until user_cache fills; still emit membership. + if account_uid: + entry = {'uid': account_uid, 'permissions': privs} + users.append(entry) + user_parts.append(f'[{account_uid}:{",".join(privs)}]') + continue + key = email.lower() + if key in seen_emails: + continue + seen_emails.add(key) + entry = {'email': email, 'permissions': privs} + if include_uids and account_uid: + entry['uid'] = account_uid + users.append(entry) + user_parts.append(f'[{email}:{",".join(privs)}]') + + teams = [] + team_parts = [] + seen_teams = set() + for t in sf.get('teams') or []: + team_uid = t.get('team_uid') or '' + if team_uid and team_uid in seen_teams: + continue + if team_uid: + seen_teams.add(team_uid) + name = (t.get('name') or '').strip() + if not name or _looks_like_uid(name): + name = _resolve_tree_team_name(params, team_uid) if team_uid else name + if not name: + if not team_uid: + continue + name = team_uid + privs = [abbr for key_name, abbr in MEMBER_PERM_KEYS.items() if t.get(key_name)] + privs = [a for a in ('MU', 'MR') if a in privs] or ['RO'] + entry = {'name': name, 'permissions': privs} + if include_uids and team_uid: + entry['uid'] = team_uid + teams.append(entry) + team_parts.append(f'[{name}:{",".join(privs)}]') + + data = { + 'user_permissions': user_permissions, + 'record_permissions': record_permissions, + } + if not user_permissions and not record_permissions: + data['record_permissions'] = ['RO'] + if users: + data['users'] = users + if teams: + data['teams'] = teams + + parts = ['default:' + ','.join(default_perms)] + if team_parts: + parts.append('teams:' + ','.join(team_parts)) + if user_parts: + parts.append('users:' + ','.join(user_parts)) + return data, f' ({"; ".join(parts)})' + + +def _classic_record_share_data(params, record_uid, *, include_uids=False): + """Classic record share perms → structured data + compact text. + + Only direct user shares are listed. Shared-folder inheritance is omitted: + the tree already places the record under its parent shared folder(s). + """ + rec = (params.record_cache or {}).get(record_uid) or {} + shares_data = rec.get('shares') or {} + users = [] + user_parts = [] + for up in shares_data.get('user_permissions') or []: + email = up.get('username') or '' + if not email: + continue + if up.get('owner'): + users.append({'email': email, 'permissions': ['OW']}) + user_parts.append(f'[{email}:OW]') + continue + if email == params.user: + continue + privs = [] + if up.get('editable'): + privs.append('CE') + if up.get('shareable'): + privs.append('CS') + if not privs: + privs = ['RO'] + users.append({'email': email, 'permissions': privs}) + user_parts.append(f'[{email}:{",".join(privs)}]') + if not users: + return None, '' + data = {'users': users} + text = f' (users:{",".join(user_parts)})' + return data, text + + +def _nsf_record_share_data(params, record_uid, *, include_uids=False): + """NSF record share perms → structured data + compact text. + + Uses warmed ``nested_share_record_share_cache``. Direct (non-inherited) + accessors win over folder-inherited rows for the same identity. When the + record ACL cache is empty, falls back to the parent NSF folder ACL. + """ + from .. import nested_share_folder as _nsf + accessors = list(_nsf.get_nsf_record_share_accessors(params, record_uid) or []) + if not accessors: + try: + parent_uids = _nsf.find_nested_share_folders_for_record(params, record_uid) or [] + except Exception as exc: + logging.debug('NSF parent folder lookup failed for %s: %s', record_uid, exc) + parent_uids = [] + for fuid in parent_uids: + data, text = _nsf_folder_share_data(params, fuid, include_uids=include_uids) + if data: + return data, text + return None, '' + + # Direct shares first so inherited folder rows do not hide them. + accessors.sort(key=lambda a: 1 if a.get('inherited') else 0) + + users = [] + teams = [] + applications = [] + user_parts = [] + team_parts = [] + app_parts = [] + seen_users = set() + seen_teams = set() + seen_apps = set() + + for a in accessors: + at = a.get('access_type') or '' + abbrev = _nsf_role_abbrev(a) + auid = a.get('access_type_uid') or a.get('accessor_uid') or '' + + if a.get('owner') or at == 'AT_OWNER': + email = (a.get('accessor_name') or '').strip() + if email and _looks_like_uid(email): + email = '' + if not email: + email = _resolve_nsf_user_email(params, { + 'username': a.get('accessor_name') or a.get('username'), + 'accessor_uid': auid, + }) + if email and email.lower() not in seen_users: + users.append({'email': email, 'permissions': ['OW']}) + user_parts.append(f'[{email}:OW]') + seen_users.add(email.lower()) + elif not email and include_uids and auid and auid not in seen_users: + users.append({'uid': auid, 'permissions': ['OW']}) + user_parts.append(f'[{auid}:OW]') + seen_users.add(auid) + continue + + if at == 'AT_TEAM': + if auid and auid in seen_teams: + continue + name = _resolve_tree_team_name(params, auid) + entry = {'name': name, 'permissions': [abbrev]} + if include_uids and auid: + entry['uid'] = auid + teams.append(entry) + team_parts.append(f'[{name}:{abbrev}]') + if auid: + seen_teams.add(auid) + continue + + if at == 'AT_APPLICATION': + if auid and auid in seen_apps: + continue + app_name = _resolve_nsf_app_name(params, auid) or (auid if include_uids else 'application') + entry = {'name': app_name, 'permissions': [abbrev]} + if include_uids and auid: + entry['uid'] = auid + applications.append(entry) + app_parts.append(f'[{app_name}:{abbrev}]') + if auid: + seen_apps.add(auid) + continue + + email = (a.get('accessor_name') or '').strip() + if email and _looks_like_uid(email): + email = '' + if not email: + email = _resolve_nsf_user_email(params, { + 'username': a.get('accessor_name') or a.get('username'), + 'accessor_uid': auid, + }) + if not email: + if include_uids and auid and auid not in seen_users: + users.append({'uid': auid, 'permissions': [abbrev]}) + user_parts.append(f'[{auid}:{abbrev}]') + seen_users.add(auid) + continue + if email.lower() in seen_users or email == params.user: + continue + users.append({'email': email, 'permissions': [abbrev]}) + user_parts.append(f'[{email}:{abbrev}]') + seen_users.add(email.lower()) + + data = {} + if users: + data['users'] = users + if teams: + data['teams'] = teams + if applications: + data['applications'] = applications + if not data: + try: + parent_uids = _nsf.find_nested_share_folders_for_record(params, record_uid) or [] + except Exception as exc: + logging.debug('NSF parent folder lookup failed for %s: %s', record_uid, exc) + parent_uids = [] + for fuid in parent_uids: + folder_data, folder_text = _nsf_folder_share_data( + params, fuid, include_uids=include_uids) + if folder_data: + return folder_data, folder_text + return None, '' + + parts = [] + if user_parts: + parts.append('users:' + ','.join(user_parts)) + if team_parts: + parts.append('teams:' + ','.join(team_parts)) + if app_parts: + parts.append('applications:' + ','.join(app_parts)) + text = f' ({"; ".join(parts)})' if parts else '' + return data, text + + +def _join_tree_path(parent_path, name): + name = name or '' + if not parent_path or parent_path == '/': + return '/' + name if name else '/' + return parent_path.rstrip('/') + '/' + name + + +def _collect_tree_share_targets(params, folder, show_records): + """Collect NSF folder UIDs and record UIDs under *folder* for ACL warming.""" + nsf_folder_uids = set() + classic_record_uids = set() + nsf_record_uids = set() + nsf_folders = getattr(params, 'nested_share_folders', {}) or {} + nsf_records = getattr(params, 'nested_share_records', {}) or {} + nsf_folder_records = getattr(params, 'nested_share_folder_records', {}) or {} + visited = set() + + def walk(node): + if isinstance(node, Record): + ruid = node.record_uid + if ruid in nsf_records: + nsf_record_uids.add(ruid) + else: + classic_record_uids.add(ruid) + return + + node_uid = node.uid if hasattr(node, 'uid') else '' + walk_key = node_uid or id(node) + if walk_key in visited: + return + visited.add(walk_key) + + is_nsf = ( + (hasattr(node, 'type') and node.type == 'nested_share_folder') + or (node_uid and node_uid in nsf_folders) + ) + if is_nsf and node_uid: + nsf_folder_uids.add(node_uid) + + dir_nodes = [] + if hasattr(node, 'subfolders'): + dir_nodes = [params.folder_cache.get(fuid) for fuid in node.subfolders if params.folder_cache.get(fuid)] + + is_root = (isinstance(node, BaseFolderNode) and (node.type == '/' or node_uid == '')) or ( + hasattr(node, 'type') and node.type == 'nested_share_folder' and not node_uid) + + if is_root and nsf_folders: + for nsf_uid, nsf_folder in nsf_folders.items(): + parent_uid = nsf_folder.get('parent_uid') + is_root_folder = ( + parent_uid is None or parent_uid == '' or parent_uid == 'root' + or parent_uid == 'AAAAAAAAAAAAAAAAAPmtNA' + or (parent_uid and parent_uid not in nsf_folders) + ) + if not is_root_folder: continue - privs = [v for k, v in perm_abbrev_lookup.items() if u.get(k)] or ['RO'] - info.append(f'[{email}:{",".join(privs)}]') - return 'users:' + ','.join(info) if info else '' + nsf_folder_uids.add(nsf_uid) + if nsf_uid in params.folder_cache: + dir_nodes.append(params.folder_cache.get(nsf_uid)) + else: + dir_nodes.append(type('FolderNode', (), { + 'uid': nsf_uid, 'name': nsf_folder.get('name', ''), + 'type': 'nested_share_folder', 'subfolders': [] + })()) + elif node_uid and nsf_folders: + for child_uid, child_folder in nsf_folders.items(): + if child_folder.get('parent_uid', '') == node_uid: + nsf_folder_uids.add(child_uid) + if child_uid in params.folder_cache: + dir_nodes.append(params.folder_cache.get(child_uid)) + else: + dir_nodes.append(type('FolderNode', (), { + 'uid': child_uid, 'name': child_folder.get('name', ''), + 'type': 'nested_share_folder', 'subfolders': [] + })()) - def get_teams_info(teams): - info = [] - for t in teams: - name = t.get('name') - privs = [v for k, v in perm_abbrev_lookup.items() if t.get(k)] or ['RO'] - info.append(f'[{name}:{",".join(privs)}]') - return 'teams:' + ','.join(info) if info else '' - - result = '' - if isinstance(node, SharedFolderNode): - sf = params.shared_folder_cache.get(node.uid) - teams_info = get_teams_info(sf.get('teams', [])) - users_info = get_users_info(sf.get('users', [])) - default_perms = [v for k, v in perm_abbrev_lookup.items() if sf.get(k)] or ['RO'] - default_perms = 'default:' + ','.join(default_perms) - user_perms = [v for k, v in perm_abbrev_lookup.items() if sf.get(k)] or ['RO'] - user_perms = 'user:' + ','.join(user_perms) - perms = [default_perms, user_perms, teams_info, users_info] - perms = [p for p in perms if p] - result = f' ({"; ".join(perms)})' if shares else '' - - return result - - def tree_node(node): + if show_records and isinstance(node, BaseFolderNode): + node_uid_for_recs = '' if node.type == '/' else node.uid + rec_uids = {rec for recs in get_contained_record_uids(params, node_uid_for_recs).values() for rec in recs} + for ruid in rec_uids: + if ruid in nsf_records: + nsf_record_uids.add(ruid) + else: + classic_record_uids.add(ruid) + if is_root: + shown = set(rec_uids) + for folder_uid, nsf_rec_uids in nsf_folder_records.items(): + if folder_uid not in nsf_folders: + for ruid in nsf_rec_uids: + if ruid not in shown: + nsf_record_uids.add(ruid) + shown.add(ruid) + all_filed = set() + for uids in nsf_folder_records.values(): + all_filed.update(uids) + for ruid in nsf_records: + if ruid not in all_filed and ruid not in shown: + nsf_record_uids.add(ruid) + elif node_uid_for_recs in nsf_folder_records: + for ruid in nsf_folder_records[node_uid_for_recs]: + nsf_record_uids.add(ruid) + elif show_records and is_nsf and node_uid and node_uid in nsf_folder_records: + # Temp NSF nodes are not BaseFolderNode; still warm their records. + for ruid in nsf_folder_records[node_uid]: + nsf_record_uids.add(ruid) + + for child in dir_nodes: + if child: + walk(child) + + walk(folder) + return nsf_folder_uids, classic_record_uids, nsf_record_uids + + +def formatted_tree(params, folder, verbose=False, show_records=False, shares=False, + nsf_shares=False, hide_shares_key=False, title=None, fmt='table'): + as_json = (fmt == 'json') + need_nsf_folders = bool(nsf_shares) + need_nsf_records = bool(nsf_shares and show_records) + need_classic_records = bool(shares and show_records) + if need_nsf_folders or need_nsf_records or need_classic_records: + nsf_folder_uids, classic_record_uids, nsf_record_uids = _collect_tree_share_targets( + params, folder, show_records) + from .. import nested_share_folder as _nsf + _nsf.warm_for_tree( + params, + nsf_folder_uids=nsf_folder_uids if need_nsf_folders else None, + classic_record_uids=classic_record_uids if need_classic_records else None, + nsf_record_uids=nsf_record_uids if need_nsf_records else None, + ) + + def print_share_permissions_key(): + lines = [ + 'Share Permissions Key:', + '======================', + ] + if shares: + lines.extend([ + 'RO = Read-Only', + 'MU = Can Manage Users', + 'MR = Can Manage Records', + 'CE = Can Edit', + 'CS = Can Share', + 'OW = Owner', + ]) + if nsf_shares: + lines.extend([ + 'OW = NSF Owner', + 'VW = NSF Viewer', + 'CT = NSF Contributor', + 'SM = NSF Share Manager', + 'CM = NSF Content Manager', + 'CSM = NSF Content + Share Manager', + 'FM = NSF Full Manager', + ]) + lines.append('======================') + print('\n'.join(lines) + '\n') + + def tree_node(node, parent_path=''): node_uid = node.record_uid if isinstance(node, Record) else (node.uid if hasattr(node, 'uid') else '') node_name = node.title if isinstance(node, Record) else (node.name if hasattr(node, 'name') else 'Unknown') - - # Check if it's a Nested Share Folder item and get proper name + is_nested_share = False if isinstance(node, Record): is_nested_share = hasattr(params, 'nested_share_records') and node.record_uid in params.nested_share_records @@ -1845,59 +2533,78 @@ def tree_node(node): is_nested_share = True elif isinstance(node, BaseFolderNode) and not isinstance(node, Record): is_nested_share = hasattr(params, 'nested_share_folders') and node_uid in params.nested_share_folders - # Get folder name from nested_share_folders if available if is_nested_share and node_uid in params.nested_share_folders: nsf_folder_name = params.nested_share_folders[node_uid].get('name', node_name) if nsf_folder_name: node_name = nsf_folder_name - - node_name = f'{node_name} ({node_uid})' if verbose else node_name - share_info = get_share_info(node) if isinstance(node, SharedFolderNode) and shares else '' - - # Format node name based on type + + base_name = node_name + if is_nested_share and not isinstance(node, Record) and node_uid in getattr(params, 'nested_share_folders', {}): + base_name = params.nested_share_folders[node_uid].get('name') or base_name + + is_vault_root = isinstance(node, BaseFolderNode) and (getattr(node, 'type', None) == '/' or not node_uid) + if is_vault_root and not isinstance(node, Record): + node_path = '/' + else: + node_path = _join_tree_path(parent_path, base_name) + + display_name = f'{node_name} ({node_uid})' if verbose else node_name + share_text = '' + share_data = None + kind = 'folder' + record_type = None + if isinstance(node, Record): + kind = 'nested_record' if is_nested_share else 'record' + record_type = getattr(node, 'record_type', None) or '' + type_label = f' [{record_type}]' if record_type else '' nsf_label = ' [Nested Record]' if is_nested_share else ' [Record]' - node_name = f'{Style.DIM}{node_name}{nsf_label}{Style.NORMAL}' + if is_nested_share and nsf_shares: + share_data, share_text = _nsf_record_share_data( + params, node_uid, include_uids=verbose) + elif (not is_nested_share) and shares: + share_data, share_text = _classic_record_share_data( + params, node_uid, include_uids=verbose) + display_name = f'{Style.DIM}{display_name}{type_label}{nsf_label}{share_text}{Style.NORMAL}' elif isinstance(node, SharedFolderNode): - node_name = f'{node_name}{Style.BRIGHT} [SHARED]{Style.NORMAL}{share_info}' + kind = 'shared_folder' + if shares: + share_data, share_text = _classic_folder_share_data( + params, params.shared_folder_cache.get(node.uid), include_uids=verbose) + display_name = f'{display_name}{Style.BRIGHT} [SHARED]{Style.NORMAL}{share_text}' elif is_nested_share: - node_name = f'{node_name}{Style.BRIGHT} [Nested Share Folder]{Style.NORMAL}' + kind = 'nested_share_folder' + if nsf_shares and node_uid: + share_data, share_text = _nsf_folder_share_data( + params, node_uid, include_uids=verbose) + display_name = f'{display_name}{Style.BRIGHT} [Nested Share Folder]{Style.NORMAL}{share_text}' dir_nodes = [] if not isinstance(node, Record): - # Get regular subfolders from folder_cache if hasattr(node, 'subfolders'): dir_nodes = [params.folder_cache.get(fuid) for fuid in node.subfolders if params.folder_cache.get(fuid)] - - # Check if this is root folder and add Nested Share Folder root-level folders + is_root = (isinstance(node, BaseFolderNode) and (node.type == '/' or node_uid == '')) or \ (hasattr(node, 'type') and node.type == 'nested_share_folder' and not node_uid) - + if is_root and hasattr(params, 'nested_share_folders') and params.nested_share_folders: - # Add all Nested Share Folders that are at root level for nsf_uid, nsf_folder in params.nested_share_folders.items(): parent_uid = nsf_folder.get('parent_uid') - # Check if this folder is at root: - # - parent_uid is None, empty string, 'root', or the special root UID - # - Also check if parent doesn't exist in nested_share_folders (orphan = root level) is_root_folder = ( - parent_uid is None or - parent_uid == '' or - parent_uid == 'root' or + parent_uid is None or + parent_uid == '' or + parent_uid == 'root' or parent_uid == 'AAAAAAAAAAAAAAAAAPmtNA' or (parent_uid and parent_uid not in params.nested_share_folders) ) if is_root_folder: - # Check if already in dir_nodes already_added = any(hasattr(n, 'uid') and n.uid == nsf_uid for n in dir_nodes if n) if not already_added: - # Check if in folder_cache first if nsf_uid in params.folder_cache: nsf_node = params.folder_cache.get(nsf_uid) if nsf_node: dir_nodes.append(nsf_node) else: - # Create a temporary folder node for Nested Share Folders not in folder_cache temp_node = type('FolderNode', (), { 'uid': nsf_uid, 'name': nsf_folder.get('name', 'Unnamed'), @@ -1905,14 +2612,11 @@ def tree_node(node): 'subfolders': [] })() dir_nodes.append(temp_node) - - # Add Nested Share Folder subfolders if this is a Nested Share Folder + elif not isinstance(node, Record) and hasattr(params, 'nested_share_folders') and node_uid: - # Find child folders for this Nested Share Folder for child_uid, child_folder in params.nested_share_folders.items(): parent_uid = child_folder.get('parent_uid', '') if parent_uid == node_uid: - # Check if already in dir_nodes already_added = any(hasattr(n, 'uid') and n.uid == child_uid for n in dir_nodes if n) if not already_added: if child_uid in params.folder_cache: @@ -1920,7 +2624,6 @@ def tree_node(node): if child_node: dir_nodes.append(child_node) else: - # Create a temporary folder node temp_node = type('FolderNode', (), { 'uid': child_uid, 'name': child_folder.get('name', 'Unnamed'), @@ -1928,25 +2631,19 @@ def tree_node(node): 'subfolders': [] })() dir_nodes.append(temp_node) - + rec_nodes = [] if show_records and isinstance(node, BaseFolderNode): node_uid_for_recs = '' if node.type == '/' else node.uid - - # Get legacy records rec_uids = {rec for recs in get_contained_record_uids(params, node_uid_for_recs).values() for rec in recs} records = [api.get_record(params, rec_uid) for rec_uid in rec_uids] records = [r for r in records if isinstance(r, Record)] rec_nodes.extend(records) - - # Add Nested Share Records for this folder + if hasattr(params, 'nested_share_folder_records'): - # For root folder, collect Nested Share Folder records that are not inside any known NSF sub-folder if is_root: nsf_folders = getattr(params, 'nested_share_folders', {}) shown_rec_uids = set(rec_uids) - # Records associated with container UIDs that are NOT real NSF sub-folders - # (includes the NSF root UID and any other non-folder containers) for folder_uid, nsf_rec_uids in params.nested_share_folder_records.items(): if folder_uid not in nsf_folders: for rec_uid in nsf_rec_uids: @@ -1955,7 +2652,6 @@ def tree_node(node): if isinstance(rec, Record): rec_nodes.append(rec) shown_rec_uids.add(rec_uid) - # Also show Nested Share Folder records that have NO folder association at all if hasattr(params, 'nested_share_records'): all_filed = set() for uids in params.nested_share_folder_records.values(): @@ -1966,7 +2662,6 @@ def tree_node(node): if isinstance(rec, Record): rec_nodes.append(rec) shown_rec_uids.add(rec_uid) - # For specific folders elif node_uid_for_recs in params.nested_share_folder_records: nsf_rec_uids = params.nested_share_folder_records[node_uid_for_recs] for rec_uid in nsf_rec_uids: @@ -1979,17 +2674,52 @@ def tree_node(node): rec_nodes.sort(key=lambda r: r.title.lower(), reverse=False) child_nodes = dir_nodes + rec_nodes - tns = [tree_node(n) for n in child_nodes] - return node_name, OrderedDict(tns) + child_path = '' if is_vault_root else node_path + child_results = [tree_node(n, child_path) for n in child_nodes] + ascii_children = OrderedDict((disp, br) for disp, br, _ in child_results) + + # Nested JSON node (omit empty children; uid only with -v) + item = {'name': base_name, 'path': node_path} + if verbose and node_uid: + item['uid'] = node_uid + item['kind'] = kind + if record_type: + item['record_type'] = record_type + if share_data: + item['share_permissions'] = share_data + json_children = [jr for _, _, jr in child_results if jr] + if json_children: + item['children'] = json_children + return display_name, ascii_children, item + + root_name, branches, json_root = tree_node(folder, '') + payload = {'tree': json_root} + if title: + payload['title'] = title + if (shares or nsf_shares) and not hide_shares_key: + key = {} + if shares: + key['classic'] = { + 'RO': 'Read-Only', 'MU': 'Can Manage Users', 'MR': 'Can Manage Records', + 'CE': 'Can Edit', 'CS': 'Can Share', 'OW': 'Owner', + } + if nsf_shares: + key['nsf'] = { + 'OW': 'Owner', 'VW': 'Viewer', 'CT': 'Contributor', 'SM': 'Share Manager', + 'CM': 'Content Manager', 'CSM': 'Content + Share Manager', 'FM': 'Full Manager', + } + payload['share_permissions_key'] = key + + if as_json: + return payload - root, branches = tree_node(folder) - tree = {root: branches} tr = LeftAligned(draw=BoxStyle(gfx=drawing.BOX_LIGHT)) - if shares and not hide_shares_key: + if (shares or nsf_shares) and not hide_shares_key: print_share_permissions_key() if title: print(title) - tree_txt = tr(tree) + tree_txt = tr({root_name: branches}) tree_txt = re.sub(r'\s+\(\)', '', tree_txt) print(tree_txt) print('') + return None diff --git a/keepercommander/nested_share_folder/__init__.py b/keepercommander/nested_share_folder/__init__.py index 1feda6a61..f92fac0a8 100644 --- a/keepercommander/nested_share_folder/__init__.py +++ b/keepercommander/nested_share_folder/__init__.py @@ -62,6 +62,11 @@ 'find_nested_share_folders_for_record', 'resolve_nested_share_record_uid', 'resolve_nested_share_folder_uid', ], + 'acl_cache': [ + 'warm_for_tree', 'warm_nsf_folder_share_cache', 'warm_nsf_record_share_cache', + 'warm_classic_record_shares', 'clear_share_caches', 'ensure_share_caches', + 'get_nsf_folder_share_accessors', 'get_nsf_record_share_accessors', + ], } _LAZY_REGISTRY = {} diff --git a/keepercommander/nested_share_folder/acl_cache.py b/keepercommander/nested_share_folder/acl_cache.py new file mode 100644 index 000000000..4d30c3d19 --- /dev/null +++ b/keepercommander/nested_share_folder/acl_cache.py @@ -0,0 +1,230 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | ' None: + if not hasattr(params, _FOLDER_SHARE_CACHE) or getattr(params, _FOLDER_SHARE_CACHE) is None: + setattr(params, _FOLDER_SHARE_CACHE, {}) + if not hasattr(params, _RECORD_SHARE_CACHE) or getattr(params, _RECORD_SHARE_CACHE) is None: + setattr(params, _RECORD_SHARE_CACHE, {}) + + +def clear_share_caches(params) -> None: + if hasattr(params, _FOLDER_SHARE_CACHE) and isinstance(getattr(params, _FOLDER_SHARE_CACHE), dict): + getattr(params, _FOLDER_SHARE_CACHE).clear() + if hasattr(params, _RECORD_SHARE_CACHE) and isinstance(getattr(params, _RECORD_SHARE_CACHE), dict): + getattr(params, _RECORD_SHARE_CACHE).clear() + + +def _chunked(items: Sequence[str], size: int) -> Iterable[List[str]]: + for i in range(0, len(items), size): + yield list(items[i:i + size]) + + +def _warm_nsf_folder_chunk(params, cache, chunk: List[str]) -> None: + """Fetch one folder-UID chunk, following pagination; leave failures uncached.""" + from .folder_api import get_folder_access_v3 + + accumulated: dict = {} + failed: Set[str] = set() + token = None + for _ in range(50): + info = get_folder_access_v3( + params, chunk, continuation_token=token, resolve_usernames=True) + for fr in info.get('results') or []: + fuid = fr.get('folder_uid') + if not fuid: + continue + if fr.get('success'): + accumulated.setdefault(fuid, []).extend(fr.get('accessors') or []) + else: + failed.add(fuid) + logging.debug('NSF folder ACL warm error for %s: %s', fuid, fr.get('error')) + if not info.get('has_more'): + break + token = info.get('continuation_token') + if token is None: + break + + for fuid, accessors in accumulated.items(): + cache[fuid] = accessors + for fuid in failed: + if fuid not in accumulated: + cache[fuid] = [] + + +def warm_nsf_folder_share_cache(params, folder_uids: Iterable[str], *, force: bool = False) -> None: + """Batch-load full NSF folder accessors into ``nested_share_folder_share_cache``. + + On transport/API exception the UIDs are left uncached so a later warm can + retry (same as record ACL warm). ``success: false`` results cache as []. + """ + ensure_share_caches(params) + cache = getattr(params, _FOLDER_SHARE_CACHE) + needed = [] + seen: Set[str] = set() + for uid in folder_uids: + if not uid or uid in seen: + continue + seen.add(uid) + if force or uid not in cache: + needed.append(uid) + if not needed: + return + + for chunk in _chunked(needed, _FOLDER_BATCH): + try: + _warm_nsf_folder_chunk(params, cache, chunk) + except Exception as exc: + logging.debug('NSF folder ACL warm failed for %s: %s', chunk, exc) + +def _store_nsf_record_accesses(cache, record_uids: Sequence[str], info: dict) -> None: + by_uid: dict = {} + for access in info.get('record_accesses', []) or []: + ruid = access.get('record_uid') + if not ruid: + continue + by_uid.setdefault(ruid, []).append(access) + forbidden = set(info.get('forbidden_records') or []) + for uid in record_uids: + if uid in forbidden: + cache[uid] = [] + elif uid in by_uid: + cache[uid] = by_uid[uid] + else: + # Successful response with no rows for this UID — real empty ACL. + cache[uid] = [] + + +def _is_throttle_error(exc: BaseException) -> bool: + text = str(exc).lower() + return 'throttl' in text or 'too many' in text or 'rate limit' in text + + +def _warm_nsf_record_chunk(params, cache, chunk: List[str], *, force: bool) -> bool: + """Warm one chunk. Returns False if further warming should stop (throttled).""" + from .record_api import get_record_accesses_v3 + + pending = [u for u in chunk if force or u not in cache] + if not pending: + return True + + try: + info = get_record_accesses_v3(params, pending) + _store_nsf_record_accesses(cache, pending, info) + return True + except Exception as exc: + if _is_throttle_error(exc): + logging.warning( + 'NSF record ACL warm throttled — stopping further record ACL fetches. ' + 'Records without a cached ACL will show parent-folder shares. (%s)', + exc) + return False + + # Binary-split once instead of N individual calls (avoids hammering / throttle). + if len(pending) > _RECORD_BATCH_MIN: + mid = len(pending) // 2 + logging.debug( + 'NSF record ACL warm failed for batch of %d; splitting: %s', + len(pending), exc) + if not _warm_nsf_record_chunk(params, cache, pending[:mid], force=force): + return False + return _warm_nsf_record_chunk(params, cache, pending[mid:], force=force) + + logging.warning('NSF record ACL warm failed for %s: %s', pending[0], exc) + # Leave uncached so folder-ACL fallback can still apply. + return True + + +def warm_nsf_record_share_cache(params, record_uids: Iterable[str], *, force: bool = False) -> None: + """Batch-load full NSF record accessors into ``nested_share_record_share_cache``. + + On failure, splits the batch rather than retrying every UID individually + (that pattern caused API floods / throttling). Throttle aborts the rest of + the warm for this call; uncached records fall back to parent folder ACL. + """ + ensure_share_caches(params) + cache = getattr(params, _RECORD_SHARE_CACHE) + needed = [] + seen: Set[str] = set() + for uid in record_uids: + if not uid or uid in seen: + continue + seen.add(uid) + if force or uid not in cache: + needed.append(uid) + if not needed: + return + + for chunk in _chunked(needed, _RECORD_BATCH): + if not _warm_nsf_record_chunk(params, cache, chunk, force=force): + break + + +def warm_classic_record_shares(params, record_uids: Iterable[str]) -> None: + """Ensure classic ``record_cache[uid]['shares']`` is populated (batched API).""" + uids = [u for u in dict.fromkeys(record_uids) if u and u in (params.record_cache or {})] + if not uids: + return + try: + api.get_record_shares(params, uids) + except Exception as exc: + logging.debug('Classic record share warm failed: %s', exc) + + +def warm_for_tree(params, + nsf_folder_uids: Optional[Iterable[str]] = None, + classic_record_uids: Optional[Iterable[str]] = None, + nsf_record_uids: Optional[Iterable[str]] = None) -> None: + """Warm all ACL caches needed for ``tree -s`` / ``tree -s -r``.""" + if nsf_folder_uids: + warm_nsf_folder_share_cache(params, nsf_folder_uids) + if nsf_record_uids: + warm_nsf_record_share_cache(params, nsf_record_uids) + if classic_record_uids: + warm_classic_record_shares(params, classic_record_uids) + + +def get_nsf_folder_share_accessors(params, folder_uid: str) -> List[dict]: + ensure_share_caches(params) + return list(getattr(params, _FOLDER_SHARE_CACHE).get(folder_uid) or []) + + +def get_nsf_record_share_accessors(params, record_uid: str) -> List[dict]: + ensure_share_caches(params) + return list(getattr(params, _RECORD_SHARE_CACHE).get(record_uid) or []) diff --git a/keepercommander/nested_share_folder/record_api.py b/keepercommander/nested_share_folder/record_api.py index f3d13979a..0d6a84832 100644 --- a/keepercommander/nested_share_folder/record_api.py +++ b/keepercommander/nested_share_folder/record_api.py @@ -328,23 +328,21 @@ def _try_decrypt_record_key(params, enc_rk, rk_type, uid): return drk -def get_record_accesses_v3(params, record_uids): - if not record_uids: - raise ValueError("At least one record UID required") - rq = record_details_pb2.RecordAccessRequest() - for uid in record_uids: - rq.recordUids.append(utils.base64_url_decode(uid)) +def _access_type_name(access_type): + try: + return folder_pb2.AccessType.Name(access_type) + except (ValueError, TypeError): + return 'AT_UNKNOWN' - rs = api.communicate_rest(params, rq, 'vault/records/v3/details/access', - rs_type=record_details_pb2.RecordAccessResponse) - result = {'record_accesses': [], 'forbidden_records': []} + +def _parse_record_access_response(rs, result): for ra in rs.recordAccesses: d = ra.data ai = ra.accessorInfo ao = { 'record_uid': utils.base64_url_encode(d.recordUid), 'accessor_name': ai.name, - 'access_type': folder_pb2.AccessType.Name(d.accessType) if hasattr(d, 'accessType') else 'UNKNOWN', + 'access_type': _access_type_name(d.accessType), 'access_type_uid': utils.base64_url_encode(d.accessTypeUid), 'owner': getattr(d, 'owner', False), 'inherited': bool(getattr(d, 'inherited', False)), @@ -361,7 +359,46 @@ def get_record_accesses_v3(params, record_uids): getattr(d.tlaProperties, 'rotateOnExpiration', False)) result['record_accesses'].append(ao) for fu in rs.forbiddenRecords: - result['forbidden_records'].append(utils.base64_url_encode(fu)) + fuid = utils.base64_url_encode(fu) + if fuid not in result['forbidden_records']: + result['forbidden_records'].append(fuid) + + +def get_record_accesses_v3(params, record_uids): + if not record_uids: + raise ValueError("At least one record UID required") + from ..proto import pagination_pb2 + + result = {'record_accesses': [], 'forbidden_records': []} + cursor = '' + page_number = 1 + use_page = False + # Guard against runaway pagination. + for _ in range(50): + rq = record_details_pb2.RecordAccessRequest() + for uid in record_uids: + rq.recordUids.append(utils.base64_url_decode(uid)) + # First request matches historical nsf-get behavior (no page). + # Only send Page on follow-ups; forcing pageSize on page 1 caused + # incomplete accessor lists for some vaults. + if use_page: + page = pagination_pb2.Page() + page.pageNumber = page_number + page.pageSize = 500 # server max + if cursor: + page.cursorToken = cursor + rq.page.CopyFrom(page) + + rs = api.communicate_rest(params, rq, 'vault/records/v3/details/access', + rs_type=record_details_pb2.RecordAccessResponse) + _parse_record_access_response(rs, result) + + page_info = rs.pageInfo if rs.HasField('pageInfo') else None + if not page_info or not page_info.hasMore: + break + cursor = page_info.cursorToken or '' + page_number = (page_info.pageNumber or page_number) + 1 + use_page = True return result diff --git a/keepercommander/nested_share_folder/sync.py b/keepercommander/nested_share_folder/sync.py index 996012fa6..69c884579 100644 --- a/keepercommander/nested_share_folder/sync.py +++ b/keepercommander/nested_share_folder/sync.py @@ -37,6 +37,10 @@ def _ensure_nested_share_folder_attrs(params): params.nested_share_record_links = {} if not hasattr(params, 'nested_share_raw_dag_data'): params.nested_share_raw_dag_data = [] + if not hasattr(params, 'nested_share_folder_share_cache'): + params.nested_share_folder_share_cache = {} + if not hasattr(params, 'nested_share_record_share_cache'): + params.nested_share_record_share_cache = {} def create_accumulator(): @@ -78,6 +82,10 @@ def clear_caches(params): params.nested_share_record_sharing_states.clear() params.nested_share_record_links.clear() params.nested_share_raw_dag_data.clear() + if hasattr(params, 'nested_share_folder_share_cache'): + params.nested_share_folder_share_cache.clear() + if hasattr(params, 'nested_share_record_share_cache'): + params.nested_share_record_share_cache.clear() # nested_share_folder_trashed_folders is intentionally NOT cleared here. # The server keeps sending trashed folders in every sync_down response # (including full/CLEAR syncs), so the trashed-UID filter must survive diff --git a/keepercommander/params.py b/keepercommander/params.py index d74297abe..7a19c6c49 100644 --- a/keepercommander/params.py +++ b/keepercommander/params.py @@ -259,6 +259,9 @@ def __init__(self, config_filename='', config=None, server='keepersecurity.com') self.nested_share_record_sharing_states = {} # record_uid -> sharing state dict self.nested_share_record_links = {} # record_uid -> list of record link dicts self.nested_share_raw_dag_data = [] # list of raw DAG entry dicts + # Full NSF ACL lists after warm (separate from sync self-rows above) + self.nested_share_folder_share_cache = {} # folder_uid -> list of accessors + self.nested_share_record_share_cache = {} # record_uid -> list of accessors self.__proxy = None self.ssh_agent = None self.unmask_all = False @@ -345,6 +348,8 @@ def clear_session(self): self.nested_share_record_sharing_states = {} self.nested_share_record_links = {} self.nested_share_raw_dag_data = [] + self.nested_share_folder_share_cache = {} + self.nested_share_record_share_cache = {} self.ws = None if self.ssh_agent: self.ssh_agent.close() diff --git a/keepercommander/service/util/parse_keeper_response.py b/keepercommander/service/util/parse_keeper_response.py index 48cbe237f..9081b81fc 100644 --- a/keepercommander/service/util/parse_keeper_response.py +++ b/keepercommander/service/util/parse_keeper_response.py @@ -241,26 +241,24 @@ def _parse_share_bracket_list( @staticmethod def _parse_tree_share_permissions(name: str) -> Optional[Dict[str, Any]]: """ - Parse shared-folder permission suffix from a tree line into structured fields. + Parse shared-folder permission suffix from ASCII tree (service ``tree``). - CLI format (from folder.formatted_tree): (default:...; user:...; teams:...; users:...) + Emits ``{"default": "...", "teams"?: ..., "users"?: ...}`` — no ``user`` + field. ``tree --format=json`` uses ``_parse_json_format_command`` instead. """ - # Ordered segments from folder.py: default, user, optional teams, optional users m = re.search( - r'\(default:([^;]+); user:([^;]+)(?:; teams:([^;]+))?(?:; users:([^)]+))?\)', + r'\(default:([^;]+)(?:; user:([^;]+))?(?:; teams:([^;]+))?(?:; users:([^)]+))?\)', name, ) if not m: return None default_val = m.group(1).strip() - user_val = m.group(2).strip() teams_seg = (m.group(3) or "").strip() users_seg = (m.group(4) or "").strip() share_permissions: Dict[str, Any] = { "default": default_val, - "user": user_val, } if teams_seg: share_permissions["teams"] = KeeperResponseParser._parse_share_bracket_list( @@ -272,9 +270,26 @@ def _parse_tree_share_permissions(name: str) -> Optional[Dict[str, Any]]: ) return share_permissions + @staticmethod + def _extract_tree_uid(name: str) -> Optional[str]: + """Return verbose-mode UID paren; ignore share-permission suffixes.""" + share_prefixes = ( + 'default:', 'user:', 'users:', 'teams:', 'folders:', + 'applications:', 'shared:', + ) + for m in re.finditer(r'\(([^)]+)\)', name): + inner = m.group(1).strip() + if any(inner.startswith(p) for p in share_prefixes): + continue + return inner + return None + @staticmethod def _parse_tree_command(response: str) -> Dict[str, Any]: - """Parse 'tree' command output into structured format.""" + """Parse ASCII ``tree`` into the legacy service-mode flat structure. + + Distinct from ``tree --format=json`` (``_parse_json_format_command``). + """ result = { "status": "success", "command": "tree", @@ -349,27 +364,32 @@ def _parse_tree_command(response: str) -> Dict[str, Any]: if not name: continue - is_record = "[Record]" in name - is_shared = "[SHARED]" in name + is_record = '[Record]' in name or '[Nested Record]' in name + is_shared = '[SHARED]' in name - # Extract UID if present (for -v flag) - uid = None - uid_match = re.search(r'\(([^)]+)\)', name) - if uid_match and not any(x in uid_match.group(1) for x in ["default:", "user:"]): - uid = uid_match.group(1) + uid = KeeperResponseParser._extract_tree_uid(name) # Extract share permissions if present (for -s flag) share_permissions = ( KeeperResponseParser._parse_tree_share_permissions(name) if is_shared else None ) - # Clean the name from all indicators + # Clean the name from indicators / share suffixes / record type tags clean_name = name - clean_name = re.sub(r' \([^)]*\) \[SHARED\] \([^)]*\)', '', clean_name) # Remove UID + SHARED + permissions - clean_name = re.sub(r' \([^)]*\) \[Record\]', '', clean_name) # Remove UID + Record - clean_name = re.sub(r' \([^)]*\) \[SHARED\]', '', clean_name) # Remove UID + SHARED - clean_name = re.sub(r' \([^)]*\)', '', clean_name) # Remove just UID - clean_name = clean_name.replace(" [Record]", "").replace(" [SHARED]", "") + clean_name = re.sub( + r' \((?:default|user|users|teams|folders|applications|shared):[^)]*\)', + '', + clean_name, + ) + clean_name = re.sub(r' \([^)]*\) \[SHARED\]', '', clean_name) + clean_name = re.sub(r' \([^)]*\)(?= \[)', '', clean_name) # UID before [tag] + clean_name = re.sub(r' \([^)]+\)$', '', clean_name) # trailing UID + for tag in ( + ' [Nested Share Folder]', ' [Nested Record]', ' [Record]', ' [SHARED]', + ): + clean_name = clean_name.replace(tag, '') + clean_name = re.sub(r' \[[A-Za-z0-9_]+\]', '', clean_name) # [login], etc. + clean_name = clean_name.strip() # Determine type item_type = "record" if is_record else "folder" @@ -1232,6 +1252,7 @@ def _filter_login_messages(response_str: str) -> str: def ensure_record_add_json_format(command: str) -> str: + """Append ``--format=json`` for record-add when missing. Not applied to tree.""" if not command.strip().startswith('record-add'): return command if '--format=json' in command or '--format json' in command: diff --git a/unit-tests/service/test_response_parser.py b/unit-tests/service/test_response_parser.py index 8512c1a98..b98a0e6b3 100644 --- a/unit-tests/service/test_response_parser.py +++ b/unit-tests/service/test_response_parser.py @@ -62,14 +62,14 @@ def test_parse_tree_command(self): self.assertEqual(result['data']['tree'][1]['path'], 'Folder1') def test_parse_tree_command_share_permissions_structured(self): - """tree -s -v: share_permissions splits default/user vs per-user list""" + """tree -s -v: share_permissions uses default only (all abbrevs), plus users list""" sample_output = """Share Permissions Key: ====================== RO = Read-Only MU = Can Manage Users ====================== My Vault - └── Shared Folder (abc123) [SHARED] (default:CE; user:CE; users:[a@x.com:RO],[b@y.com:MU,MR]) + └── Shared Folder (abc123) [SHARED] (default:CE; users:[a@x.com:RO],[b@y.com:MU,MR]) """ result = KeeperResponseParser._parse_tree_command(sample_output) self.assertEqual(result['data']['share_permissions_key'][:2], ['RO = Read-Only', 'MU = Can Manage Users']) @@ -77,13 +77,56 @@ def test_parse_tree_command_share_permissions_structured(self): self.assertTrue(entry['shared']) sp = entry['share_permissions'] self.assertEqual(sp['default'], 'CE') - self.assertEqual(sp['user'], 'CE') + self.assertNotIn('user', sp) self.assertEqual(len(sp['users']), 2) self.assertEqual(sp['users'][0]['username'], 'a@x.com') self.assertEqual(sp['users'][0]['permissions'], 'RO') self.assertEqual(sp['users'][1]['username'], 'b@y.com') self.assertEqual(sp['users'][1]['permissions'], 'MU,MR') + def test_parse_tree_share_permissions_ignores_legacy_user_segment(self): + """Legacy user: segment is ignored; only default is kept.""" + line = 'Shared Folder [SHARED] (default:MU,CE; user:MU,CE; users:[a@x.com:RO])' + sp = KeeperResponseParser._parse_tree_share_permissions(line) + self.assertEqual(sp['default'], 'MU,CE') + self.assertNotIn('user', sp) + self.assertEqual(sp['users'][0]['username'], 'a@x.com') + + def test_parse_tree_does_not_treat_share_suffix_as_uid(self): + sample = ( + 'My Vault\n' + ' └── Folder (abcUID12345678901234) [SHARED] ' + '(default:RO; users:[a@x.com:RO])\n' + ' └── Rec [login] [Nested Record] (users:[me@x.com:OW])\n' + ) + result = KeeperResponseParser._parse_tree_command(sample) + folder = result['data']['tree'][0] + self.assertEqual(folder['name'], 'Folder') + self.assertEqual(folder['uid'], 'abcUID12345678901234') + self.assertEqual(folder['share_permissions']['default'], 'RO') + rec = result['data']['tree'][1] + self.assertEqual(rec['name'], 'Rec') + self.assertEqual(rec['type'], 'record') + self.assertNotIn('uid', rec) + + def test_tree_format_json_uses_native_json_parser(self): + """tree --format=json must not use the ASCII tree parser.""" + self.assertEqual( + KeeperResponseParser._find_parser_method('tree --format=json'), + '_parse_json_format_command', + ) + self.assertEqual( + KeeperResponseParser._find_parser_method('tree'), + '_parse_tree_command', + ) + native = '{"tree": {"name": "My Vault", "kind": "folder", "children": []}}' + result = KeeperResponseParser._parse_json_format_command( + 'tree --format=json', native + ) + self.assertEqual(result['status'], 'success') + self.assertEqual(result['data']['tree']['kind'], 'folder') + self.assertNotIn('level', result['data']['tree']) + def test_parse_mkdir_command(self): """Test parsing of 'mkdir' command output""" diff --git a/unit-tests/test_command_enterprise.py b/unit-tests/test_command_enterprise.py index ac8a0bbde..a7d17b9b0 100644 --- a/unit-tests/test_command_enterprise.py +++ b/unit-tests/test_command_enterprise.py @@ -55,6 +55,29 @@ def test_enterprise_info_command(self): cmd = enterprise.EnterpriseInfoCommand() cmd.execute(params, verbose=True) + def test_enterprise_info_users_verbose_returns_ids(self): + """With -v, node/teams/roles columns should be IDs; without -v, names.""" + params = get_connected_params() + api.query_enterprise(params) + cmd = enterprise.EnterpriseInfoCommand() + columns = 'name,node,teams,roles' + + report = cmd.execute( + params, users=True, format='json', columns=columns, quiet=True) + users = json.loads(report) + user1 = next(u for u in users if u['user_id'] == ent_env.user1_id) + self.assertEqual(user1['node'], 'Enterprise 1') + self.assertEqual(user1['teams'], [ent_env.team1_name]) + self.assertEqual(user1['roles'], [ent_env.role1_name]) + + report = cmd.execute( + params, users=True, format='json', columns=columns, verbose=True, quiet=True) + users = json.loads(report) + user1 = next(u for u in users if u['user_id'] == ent_env.user1_id) + self.assertEqual(user1['node'], str(ent_env.node1_id)) + self.assertEqual(user1['teams'], [ent_env.team1_uid]) + self.assertEqual(user1['roles'], [str(ent_env.role1_id)]) + def test_enterprise_add_user(self): params = get_connected_params() api.query_enterprise(params) diff --git a/unit-tests/test_nsf_acl_cache.py b/unit-tests/test_nsf_acl_cache.py new file mode 100644 index 000000000..31d67354b --- /dev/null +++ b/unit-tests/test_nsf_acl_cache.py @@ -0,0 +1,473 @@ +# _ __ +# | |/ /___ ___ _ __ ___ _ _ ® +# | '