From 1c48ae204317bfcd7df490c86cb5e61ffa2f34f4 Mon Sep 17 00:00:00 2001 From: MarcSkovMadsen Date: Sun, 12 Nov 2023 06:24:20 +0000 Subject: [PATCH 1/5] add comment --- param/parameters.py | 134 +++++++++++++++++++++++++++++++------------- 1 file changed, 94 insertions(+), 40 deletions(-) diff --git a/param/parameters.py b/param/parameters.py index ed11b53bd..3f8f6ea9e 100644 --- a/param/parameters.py +++ b/param/parameters.py @@ -15,6 +15,7 @@ parameter types (e.g. Number), and also imports the definition of Parameters and Parameterized classes. """ +from __future__ import annotations import numbers import os.path @@ -30,6 +31,7 @@ from collections import OrderedDict from contextlib import contextmanager +from functools import lru_cache from .parameterized import ( Parameterized, Parameter, ParameterizedFunction, ParamOverrides, String, @@ -2598,6 +2600,81 @@ def _validate_value(self, val, allow_None): # - use resolve_path(path_to_file=False) for paths to existing folders to be read, # and normalize_path() for paths to new files to be written. +def _get_default_search_paths()->typing.Tuple[str]: + # Its 10x faster to pickle a string than a Path. + # Thus we use os.getcwd instead of pathlib.Path.cwd + return (os.getcwd(), ) + +@lru_cache +def _resolve_path_fast( + path: str, + search_paths: typing.Tuple[str|pathlib.Path], + path_to_file: bool|None=None +)->str: + + _path = pathlib.Path(os.path.normpath(path)) + ftype = "File" if path_to_file is True \ + else "Folder" if path_to_file is False else "Path" + + if _path.is_absolute(): + if ((path_to_file is None and _path.exists()) or + (path_to_file is True and _path.is_file()) or + (path_to_file is False and _path.is_dir())): + return str(_path) + raise OSError(f"{ftype} '{path}' was not found.") + + paths_tried = [] + for prefix in search_paths: + try_path = (pathlib.Path(prefix) / _path).resolve() + + if ((path_to_file is None and try_path.exists()) or + (path_to_file is True and try_path.is_file()) or + (path_to_file is False and try_path.is_dir())): + return str(try_path.absolute()) + + paths_tried.append(str(try_path)) + + raise OSError( + f"{ftype} {_path.name} was not found in the following place(s): {str(paths_tried)}." + ) + +def _resolve_path( + path: str, + search_paths: typing.List[str|pathlib.Path]|None=None, + path_to_file: bool|None=None +)->str: + """ + Find the path to an existing file, searching the paths specified + in the search_paths parameter if the filename is not absolute, and + converting a UNIX-style path to the current OS's format if + necessary. + + To turn a supplied relative path into an absolute one, the path is + appended to paths in the search_paths parameter, in order, until + the file is found. + + An IOError is raised if the file is not found. + + Similar to Python's os.path.abspath(), except more search paths + than just os.getcwd() can be used, and the file must exist. + + Parameters + ---------- + search_paths : tuple, default=(os.getcwd(),) + Tuple of paths to search the path from + check_exists: boolean, default=True + If True (default) the path must exist on instantiation and set, + otherwise the path can optionally exist. + """ + if not search_paths: + _search_paths = _get_default_search_paths() + else: + _search_paths = tuple(search_paths) + return _resolve_path_fast(path, _search_paths, path_to_file) + +def _get_default_search_paths_as_list()->typing.List[str]: + return [str(path) for path in _get_default_search_paths()] + class resolve_path(ParameterizedFunction): """ Find the path to an existing file, searching the paths specified @@ -2615,7 +2692,7 @@ class resolve_path(ParameterizedFunction): than just os.getcwd() can be used, and the file must exist. """ - search_paths = List(default=[os.getcwd()], pickle_default_value=False, doc=""" + search_paths = List(default=list(_get_default_search_paths_as_list()), pickle_default_value=False, doc=""" Prepended to a non-relative path, in order, until a file is found.""") @@ -2625,35 +2702,11 @@ class resolve_path(ParameterizedFunction): 'Folder'. If None, the path may point to *either* a 'File' *or* a 'Folder'.""") + def __call__(self, path, **params): p = ParamOverrides(self, params) - path = os.path.normpath(path) - ftype = "File" if p.path_to_file is True \ - else "Folder" if p.path_to_file is False else "Path" - - if not p.search_paths: - p.search_paths = [os.getcwd()] - - if os.path.isabs(path): - if ((p.path_to_file is None and os.path.exists(path)) or - (p.path_to_file is True and os.path.isfile(path)) or - (p.path_to_file is False and os.path.isdir( path))): - return path - raise OSError(f"{ftype} '{path}' not found.") - - else: - paths_tried = [] - for prefix in p.search_paths: - try_path = os.path.join(os.path.normpath(prefix), path) - - if ((p.path_to_file is None and os.path.exists(try_path)) or - (p.path_to_file is True and os.path.isfile(try_path)) or - (p.path_to_file is False and os.path.isdir( try_path))): - return try_path - paths_tried.append(try_path) - - raise OSError(ftype + " " + os.path.split(path)[1] + " was not found in the following place(s): " + str(paths_tried) + ".") + return _resolve_path(path, p.search_paths, p.path_to_file) # PARAM3_DEPRECATION @@ -2737,10 +2790,11 @@ def __init__(self, default=Undefined, *, search_paths=Undefined, check_exists=Un raise ValueError("'check_exists' attribute value must be a boolean") self.check_exists = check_exists super().__init__(default,**params) + self._validate(self.default) def _resolve(self, path): - return resolve_path(path, path_to_file=None, search_paths=self.search_paths) + return _resolve_path(path=path, search_paths=self.search_paths) def _validate(self, val): if val is None: @@ -2760,17 +2814,17 @@ def __get__(self, obj, objtype): Return an absolute, normalized path (see resolve_path). """ raw_path = super().__get__(obj,objtype) + if raw_path is None: - path = None - else: - try: - path = self._resolve(raw_path) - except OSError: - if self.check_exists: - raise - else: - path = raw_path - return path + return None + + try: + return self._resolve(raw_path) + except OSError: + if self.check_exists: + raise + else: + return raw_path def __getstate__(self): # don't want to pickle the search_paths @@ -2802,7 +2856,7 @@ class Filename(Path): """ def _resolve(self, path): - return resolve_path(path, path_to_file=True, search_paths=self.search_paths) + return _resolve_path(path, path_to_file=True, search_paths=self.search_paths) class Foldername(Path): @@ -2824,7 +2878,7 @@ class Foldername(Path): """ def _resolve(self, path): - return resolve_path(path, path_to_file=False, search_paths=self.search_paths) + return _resolve_path(path, path_to_file=False, search_paths=self.search_paths) #----------------------------------------------------------------------------- # Color From ff0ae764f9de3129abdae10bfe3577369fb4f76a Mon Sep 17 00:00:00 2001 From: MarcSkovMadsen Date: Sun, 12 Nov 2023 17:46:47 +0000 Subject: [PATCH 2/5] revert from pathlib to os due to speed --- param/parameters.py | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/param/parameters.py b/param/parameters.py index 3f8f6ea9e..e3fd3a312 100644 --- a/param/parameters.py +++ b/param/parameters.py @@ -2601,41 +2601,38 @@ def _validate_value(self, val, allow_None): # and normalize_path() for paths to new files to be written. def _get_default_search_paths()->typing.Tuple[str]: - # Its 10x faster to pickle a string than a Path. - # Thus we use os.getcwd instead of pathlib.Path.cwd return (os.getcwd(), ) - @lru_cache def _resolve_path_fast( path: str, search_paths: typing.Tuple[str|pathlib.Path], path_to_file: bool|None=None -)->str: - _path = pathlib.Path(os.path.normpath(path)) +): + path = os.path.normpath(path) ftype = "File" if path_to_file is True \ else "Folder" if path_to_file is False else "Path" - if _path.is_absolute(): - if ((path_to_file is None and _path.exists()) or - (path_to_file is True and _path.is_file()) or - (path_to_file is False and _path.is_dir())): - return str(_path) - raise OSError(f"{ftype} '{path}' was not found.") + if os.path.isabs(path): + if ((path_to_file is None and os.path.exists(path)) or + (path_to_file is True and os.path.isfile(path)) or + (path_to_file is False and os.path.isdir( path))): + return path + raise OSError(f"{ftype} '{path}' not found.") paths_tried = [] for prefix in search_paths: - try_path = (pathlib.Path(prefix) / _path).resolve() + try_path = os.path.join(os.path.normpath(prefix), path) - if ((path_to_file is None and try_path.exists()) or - (path_to_file is True and try_path.is_file()) or - (path_to_file is False and try_path.is_dir())): - return str(try_path.absolute()) + if ((path_to_file is None and os.path.exists(try_path)) or + (path_to_file is True and os.path.isfile(try_path)) or + (path_to_file is False and os.path.isdir( try_path))): + return try_path - paths_tried.append(str(try_path)) + paths_tried.append(try_path) raise OSError( - f"{ftype} {_path.name} was not found in the following place(s): {str(paths_tried)}." + f"{ftype} {os.path.split(path)[1]} was not found in the following place(s): {str(paths_tried)}." ) def _resolve_path( From 1ce1cafb47eb2af832796b53cbdf77401c19f7e9 Mon Sep 17 00:00:00 2001 From: MarcSkovMadsen Date: Sun, 12 Nov 2023 18:06:26 +0000 Subject: [PATCH 3/5] add asv benchmarks --- benchmarks/benchmarks/benchmarks.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/benchmarks/benchmarks/benchmarks.py b/benchmarks/benchmarks/benchmarks.py index 3619604fb..fa44c8c10 100644 --- a/benchmarks/benchmarks/benchmarks.py +++ b/benchmarks/benchmarks/benchmarks.py @@ -615,3 +615,21 @@ def foo0(self): pass def time_trigger(self): self.p.x0 += 1 + +class ResolvePathSuite: + + def setup(self): + class P(param.Parameterized): + x0 = param.Path("benchmarks/benchmarks/benchmarks.py") + + self.P = P + + def resolve_new(self): + for _ in range(1000): + p = self.P() + _ = p.x0 + + def resolve_same(self): + p = self.P() + for _ in range(1000): + _ = p.x0 From 70e09b106d1490bd0230f8d9bd958d975dff98b3 Mon Sep 17 00:00:00 2001 From: MarcSkovMadsen Date: Sun, 12 Nov 2023 18:13:23 +0000 Subject: [PATCH 4/5] get benchmarks working --- benchmarks/benchmarks/benchmarks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/benchmarks/benchmarks.py b/benchmarks/benchmarks/benchmarks.py index fa44c8c10..3d8384d5d 100644 --- a/benchmarks/benchmarks/benchmarks.py +++ b/benchmarks/benchmarks/benchmarks.py @@ -624,12 +624,12 @@ class P(param.Parameterized): self.P = P - def resolve_new(self): + def time_resolve_new(self): for _ in range(1000): p = self.P() _ = p.x0 - def resolve_same(self): + def time_resolve_same(self): p = self.P() for _ in range(1000): _ = p.x0 From 436823aeba62e4e42600bdbda91853607406ed09 Mon Sep 17 00:00:00 2001 From: MarcSkovMadsen Date: Sun, 12 Nov 2023 18:19:48 +0000 Subject: [PATCH 5/5] fix oneline issue --- param/parameters.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/param/parameters.py b/param/parameters.py index e3fd3a312..883dfdc2e 100644 --- a/param/parameters.py +++ b/param/parameters.py @@ -2689,7 +2689,7 @@ class resolve_path(ParameterizedFunction): than just os.getcwd() can be used, and the file must exist. """ - search_paths = List(default=list(_get_default_search_paths_as_list()), pickle_default_value=False, doc=""" + search_paths = List(default=_get_default_search_paths_as_list(), pickle_default_value=False, doc=""" Prepended to a non-relative path, in order, until a file is found.""") @@ -2820,8 +2820,8 @@ def __get__(self, obj, objtype): except OSError: if self.check_exists: raise - else: - return raw_path + + return raw_path def __getstate__(self): # don't want to pickle the search_paths