Auto-detect the configured Python version and look in PATH as a fallback on Linux

CHANGELOG.md

@@ -14,6 +14,7 @@ no overlap.
 - #1077: Fixed error handling of dependency resolution to correctly report what the offending
 modules are.
 - #1187: Remove extra path entries in output of %IPM.Storage.ResourceReference:ResolveChildren() that broke unguarded downstream callers.
+- #1191: Fixed issue where configured Python version would not be used automatically during calls to pip
 
 ## [0.10.7] - 2026-05-29
 

src/cls/IPM/Lifecycle/Base.cls

@@ -816,6 +816,9 @@ ClassMethod ResolvePipCaller(ByRef pParams) As %List
     return ..DetectPipCaller(tUseStandalonePip, $get(pParams("Verbose"), 0))
 }
 
+/// Auto-detects a pip caller when PipCaller is not configured. Tries, in order:
+/// flexible Python (PythonRuntimeLibrary), irispip.exe (Windows),
+/// python3/python in PATH, pip3/pip in PATH. Throws if nothing is found.
 ClassMethod DetectPipCaller(
 	pUseStandalonePip As %Boolean,
 	pVerbose As %Boolean = 0) As %List
@@ -825,31 +828,9 @@ ClassMethod DetectPipCaller(
     }
 
     // First try to detect flexible python (in 2024.2 and later)
-    // This is a hack that doesn't always work because the python interpreter may not be in the same directory as the so/dylib/dll.
-    if $system.CLS.IsMthd("%SYS.Python","GetPythonInfo") {
-        if pVerbose {
-            write !, "Attempting to find flexible python... "
-        }
-        do ##class(%SYS.Python).GetPythonInfo(.info)
-        if $data(info("CPF_PythonRuntimeLibrary"), tPyDylib) && (tPyDylib '= "") {
-            // TODO: try `../bin/python3` or `../bin/python` in case the .so is in subfolder `lib`
-            // TODO: try to find `pip3` or `pip` if pUseStandalonePip is 1
-            for filename = "python3", "python" {
-                set tInterpreter = ##class(%File).GetDirectory(tPyDylib, 1) _ filename
-                if $$$isWINDOWS {
-                    set tInterpreter = tInterpreter_".exe"
-                }
-                if ##class(%File).Exists(tInterpreter) {
-                    if pVerbose{
-                        write "Success!"
-                    }
-                    return $listbuild(tInterpreter, "-m", "pip")
-                }
-                if pVerbose {
-                    write "Not Found"
-                }
-            }
-        }
+    set result = ..DetectFlexiblePythonCaller(pUseStandalonePip, pVerbose)
+    if result '= "" {
+        return result
     }
 
     // For windows, try to find irispip.exe (in 2024.1 and earlier)
@@ -869,6 +850,9 @@ ClassMethod DetectPipCaller(
         }
     }
 
+    // UseStandalonePip is a tristate: 0 = python only, 1 = pip only, "" = try both.
+    // The '=1 / '=0 checks implement that: when unset, both blocks run.
+
     // Unless UseStandalonePip is set to 1, try to find python3 or python
     if pUseStandalonePip '= 1 {
         if pVerbose {
@@ -917,7 +901,128 @@ ClassMethod DetectPipCaller(
         }
     }
 
-    throw ##class(%Exception.General).%New("Could not find a suitable pip caller. Consider setting UseStandalonePip and PipCaller")
+    throw ##class(%Exception.General).%New("Could not find a suitable pip caller. Run: zpm ""config set PipCaller <path>"" and zpm ""config set UseStandalonePip 0"" (python executable) or 1 (pip executable)")
+}
+
+/// Looks for executables by searching names in priority order across all dirs.
+/// Returns the full path of the highest-priority name found, or "" if none exists.
+ClassMethod FindExecutableInDirs(dirs As %List, names As %List) As %String
+{
+    set namePtr = 0
+    while $listnext(names, namePtr, name) {
+        set dirPtr = 0
+        while $listnext(dirs, dirPtr, dir) {
+            set path = dir _ name
+            if $$$isWINDOWS {
+                set path = path _ ".exe"
+            }
+            if ##class(%File).Exists(path) {
+                return path
+            }
+        }
+    }
+    return ""
+}
+
+/// Attempts to find a pip caller using PythonRuntimeLibrary from iris.cpf (IRIS 2024.2+).
+/// Returns a $list command suitable for RunCommand, or "" if unavailable or not found.
+ClassMethod DetectFlexiblePythonCaller(useStandalonePip As %Boolean, verbose As %Boolean = 0) As %List
+{
+    if '$system.CLS.IsMthd("%SYS.Python", "GetPythonInfo") {
+        return ""
+    }
+
+    if verbose {
+        write !, "Attempting to find flexible python... "
+    }
+
+    do ##class(%SYS.Python).GetPythonInfo(.info)
+    if '$data(info("CPF_PythonRuntimeLibrary"), pyDylib) || (pyDylib = "") {
+        return ""
+    }
+
+    set dylibDir = ##class(%File).GetDirectory(pyDylib, 1)
+    set versionedName = ..VersionedPythonName(pyDylib)
+
+    // Search the .so directory, the sibling bin/, and all PATH directories.
+    // The sibling bin/ covers distros where the runtime lib and executables are
+    // co-located under lib/ (e.g. RHEL: /usr/lib64/libpython3.12.so + /usr/lib64/python3.12).
+    // PATH covers distros with a multiarch layout where the .so and executables
+    // are not co-located (e.g. Ubuntu: /usr/lib/x86_64-linux-gnu/libpython3.12.so.1.0
+    // but /usr/bin/python3.12).
+    set searchDirs = $listbuild(dylibDir, ##class(%File).NormalizeDirectory(##class(%File).NormalizeFilename("../bin", dylibDir)))
+    set envPath = $system.Util.GetEnviron("PATH")
+    for i=1:1:$length(envPath, ":") {
+        set dir = $piece(envPath, ":", i)
+        if dir '= "" {
+            set searchDirs = searchDirs _ $listbuild(##class(%File).NormalizeDirectory(dir))
+        }
+    }
+
+    if useStandalonePip '= 1 {
+        set pythonNames = $listbuild("python3", "python")
+        if versionedName '= "" {
+            set pythonNames = $listbuild(versionedName) _ pythonNames
+        }
+        set interpreter = ..FindExecutableInDirs(searchDirs, pythonNames)
+        if interpreter '= "" {
+            if verbose {
+                write "Success!"
+            }
+            return $listbuild(interpreter, "-m", "pip")
+        }
+        if verbose {
+            write !, "Python interpreter not found in flexible Python directories"
+        }
+    }
+
+    if useStandalonePip '= 0 {
+        set pipNames = $listbuild("pip3", "pip")
+        if versionedName '= "" {
+            set pipNames = $listbuild("pip" _ $piece(versionedName, "python", 2)) _ pipNames
+        }
+        set pipExec = ..FindExecutableInDirs(searchDirs, pipNames)
+        if pipExec '= "" {
+            if verbose {
+                write "Success!"
+            }
+            return $listbuild(pipExec)
+        }
+        if verbose {
+            write !, "pip not found in flexible Python directories"
+        }
+    }
+
+    return ""
+}
+
+/// Given a path to a Python runtime shared library, returns the versioned executable
+/// name (e.g. "python3.12") derived from the filename, or "" if one cannot be determined.
+/// Only handles the Linux .so convention (libpythonX.Y.so[.1.0]); other platforms
+/// (macOS .dylib, Windows .dll) return "" and fall through to unversioned PATH search.
+ClassMethod VersionedPythonName(dylib As %String) As %String
+{
+    // Strip the directory to work with just the filename
+    set filename = ##class(%File).GetFilename(dylib)
+
+    // libpythonX.Y.so[.anything] -> "libpythonX.Y"
+    set base = $piece(filename, ".so")
+    if base = filename {
+        // No ".so" in the name — not a Linux shared library we can parse
+        return ""
+    }
+
+    // "libpythonX.Y" -> "pythonX.Y"
+    if $extract(base, 1, 3) '= "lib" {
+        return ""
+    }
+    set name = $extract(base, 4, *)
+
+    // Require the result to look like pythonX.Y (major.minor, both numeric)
+    if '$match(name, "^python[0-9]+\.[0-9]+$") {
+        return ""
+    }
+    return name
 }
 
 Method %Validate(ByRef pParams) As %Status

tests/unit_tests/Test/PM/Unit/EmbeddedPython.cls

@@ -4,106 +4,106 @@ Class Test.PM.Unit.EmbeddedPython Extends %UnitTest.TestCase
 Method TestCompareVersionTotalOrder()
 {
     // format: (ver1, ver2, result), where result = 0 if ver1 = ver2; result = -1 if ver1 < ver2; result = 1 if ver1 > ver2
-    Set list = $ListBuild(
-        $ListBuild("1.2.3-alpha+build", "1.2.3-alpha+build", 0),
-        $ListBuild("1.2.3", "1.2.3-alpha+build", 1),
-        $ListBuild("1.2.3-alpha+build", "1.2.3+build", -1),
-        $ListBuild("1.2.3-alpha+build", "4.5.6+build", -1),
-        $ListBuild("4.5.6", "4.5.6+build", -1)
+    set list = $listbuild(
+        $listbuild("1.2.3-alpha+build", "1.2.3-alpha+build", 0),
+        $listbuild("1.2.3", "1.2.3-alpha+build", 1),
+        $listbuild("1.2.3-alpha+build", "1.2.3+build", -1),
+        $listbuild("1.2.3-alpha+build", "4.5.6+build", -1),
+        $listbuild("4.5.6", "4.5.6+build", -1)
     )
 
-    Set ptr = 0
-    While $ListNext(list, ptr, tuple) {
-        Set $ListBuild(ver1, ver2, expected) = tuple
-        For reverse = 0, 1 {
-            If reverse {
-                Set $Listbuild(ver2, ver1, expected) = $Listbuild(ver1, ver2, -expected)
+    set ptr = 0
+    while $listnext(list, ptr, tuple) {
+        set $listbuild(ver1, ver2, expected) = tuple
+        for reverse = 0, 1 {
+            if reverse {
+                set $listbuild(ver2, ver1, expected) = $listbuild(ver1, ver2, -expected)
             }
 
-            Set output = ##class(%IPM.Utils.EmbeddedPython).CompareVersionTotalOrder(ver1, ver2)
-            If '$$$AssertEquals(output, expected) {
-                Do $$$LogMessage("Failed for "_ver1_" and "_ver2_" with expected "_expected_" but got "_output)
+            set output = ##class(%IPM.Utils.EmbeddedPython).CompareVersionTotalOrder(ver1, ver2)
+            if '$$$AssertEquals(output, expected) {
+                do $$$LogMessage("Failed for "_ver1_" and "_ver2_" with expected "_expected_" but got "_output)
             }
         }
     }
 }
 
 Method TestSortVersions()
 {
-    Set builtins = ##class(%SYS.Python).Builtins()
-    Set versions = $ListBuild(
+    set builtins = ##class(%SYS.Python).Builtins()
+    set versions = $listbuild(
         "1.3.0",
         "1.2.3+build",
         "2.3.1",
         "1.2.4",
         "1.2.3",
         "1.2.3-alpha+build"
     )
-    Set versions = ##class(%SYS.Python).ToList(versions)
+    set versions = ##class(%SYS.Python).ToList(versions)
 
-    Set expected = $ListBuild(
+    set expected = $listbuild(
         "1.2.3-alpha+build",
         "1.2.3",
         "1.2.3+build",
         "1.2.4",
         "1.3.0",
         "2.3.1"
     )
-    Set expected = ##class(%SYS.Python).ToList(expected)
-    Set output = ##class(%IPM.Utils.EmbeddedPython).SortVersions(versions)
-    Do ..AssertPythonListsEqual(output, expected)
+    set expected = ##class(%SYS.Python).ToList(expected)
+    set output = ##class(%IPM.Utils.EmbeddedPython).SortVersions(versions)
+    do ..AssertPythonListsEqual(output, expected)
 
-    Set expected = builtins.list(builtins.reversed(expected))
-    Set output = ##class(%IPM.Utils.EmbeddedPython).SortVersions(versions, 1)
-    Do ..AssertPythonListsEqual(output, expected)
+    set expected = builtins.list(builtins.reversed(expected))
+    set output = ##class(%IPM.Utils.EmbeddedPython).SortVersions(versions, 1)
+    do ..AssertPythonListsEqual(output, expected)
 }
 
 Method TestSortOCITags()
 {
-    Set builtins = ##class(%SYS.Python).Builtins()
+    set builtins = ##class(%SYS.Python).Builtins()
 
-    Set tags = $ListBuild(
-        "4.5.6_build", 
-        "1.2.3-alpha_build__2024.1", 
-        "1.0.0_build__2024.2", 
+    set tags = $listbuild(
+        "4.5.6_build",
+        "1.2.3-alpha_build__2024.1",
+        "1.0.0_build__2024.2",
         "4.5.6",
         "1.0.0-alpha_build__2024.2"
     )
-    Set tags = ##class(%SYS.Python).ToList(tags)
+    set tags = ##class(%SYS.Python).ToList(tags)
 
-    Set expected = $ListBuild(
+    set expected = $listbuild(
         "1.0.0-alpha_build__2024.2",
         "1.0.0_build__2024.2",
         "1.2.3-alpha_build__2024.1",
         "4.5.6",
         "4.5.6_build"
     )
-    Set expected = ##class(%SYS.Python).ToList(expected)
+    set expected = ##class(%SYS.Python).ToList(expected)
 
-    Set output = ##class(%IPM.Utils.EmbeddedPython).SortOCITags(tags)
-    Do ..AssertPythonListsEqual(output, expected) 
+    set output = ##class(%IPM.Utils.EmbeddedPython).SortOCITags(tags)
+    do ..AssertPythonListsEqual(output, expected)
 
-    Set expected = builtins.list(builtins.reversed(expected))
+    set expected = builtins.list(builtins.reversed(expected))
 
-    Set output = ##class(%IPM.Utils.EmbeddedPython).SortOCITags(tags, 1)
-    Do ..AssertPythonListsEqual(output, expected)
+    set output = ##class(%IPM.Utils.EmbeddedPython).SortOCITags(tags, 1)
+    do ..AssertPythonListsEqual(output, expected)
 }
 
 Method TestFromPythonList(list As %SYS.Python) As %List
 {
-    Set original = $ListBuild($ListBuild(1, 2, 3), $ListBuild(4, 5, 6), $ListBuild($ListBuild(7, 8, 9), 10, 11))
-    Set input = ##class(%SYS.Python).ToList(original)
-    Set output = ##class(%IPM.Utils.EmbeddedPython).FromPythonList(input)
-    Do $$$AssertEquals(output, original)
+    set original = $listbuild($listbuild(1, 2, 3), $listbuild(4, 5, 6), $listbuild($listbuild(7, 8, 9), 10, 11))
+    set input = ##class(%SYS.Python).ToList(original)
+    set output = ##class(%IPM.Utils.EmbeddedPython).FromPythonList(input)
+    do $$$AssertEquals(output, original)
 }
 
 Method AssertPythonListsEqual(
 	list1 As %SYS.Python,
 	list2 As %SYS.Python) As %Boolean
 {
-    Set list1 = ##class(%IPM.Utils.EmbeddedPython).FromPythonList(list1)
-    Set list2 = ##class(%IPM.Utils.EmbeddedPython).FromPythonList(list2)
-    Return $$$AssertEquals(list1, list2)
+    set list1 = ##class(%IPM.Utils.EmbeddedPython).FromPythonList(list1)
+    set list2 = ##class(%IPM.Utils.EmbeddedPython).FromPythonList(list2)
+    return $$$AssertEquals(list1, list2)
 }
 
 }

tests/unit_tests/Test/PM/Unit/LifecycleBase.cls

@@ -0,0 +1,29 @@
+Class Test.PM.Unit.LifecycleBase Extends %UnitTest.TestCase
+{
+
+Method TestVersionedPythonName()
+{
+    // Each entry: $lb(dylib path, expected versioned name or "")
+    // Focus: Linux .so paths. macOS .dylib and Windows .dll are not handled
+    // (versioned detection falls through to unversioned PATH search on those platforms).
+    set cases = $listbuild(
+        $listbuild("/usr/lib64/libpython3.12.so.1.0", "python3.12"),
+        $listbuild("/usr/lib64/libpython3.12.so",     "python3.12"),
+        $listbuild("/usr/lib/libpython3.11.so.1.0",   "python3.11"),
+        $listbuild("/usr/lib/libpython3.so",          ""),
+        $listbuild("/usr/lib/libpython.so",           ""),
+        $listbuild("/usr/local/Frameworks/Python.framework/libpython3.12.dylib", ""),
+        $listbuild("",                                "")
+    )
+
+    set ptr = 0
+    while $listnext(cases, ptr, case) {
+        set $listbuild(dylib, expected) = case
+        set actual = ##class(%IPM.Lifecycle.Base).VersionedPythonName(dylib)
+        if '$$$AssertEquals(actual, expected) {
+            do $$$LogMessage("Failed for dylib='" _ dylib _ "': expected='" _ expected _ "' got='" _ actual _ "'")
+        }
+    }
+}
+
+}