Skip to content

Fix wrong retcodes from concurrent jobs on a threaded minion #69755

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ggiesen wants to merge 1 commit into
base: 3008.x
Choose a base branch
from
+261 −36
Open

Fix wrong retcodes from concurrent jobs on a threaded minion #69755

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog/61830.fixed.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fixed threaded minion jobs (``multiprocessing: False``) reporting the wrong retcode, including a failed command returning success. Every job rebuilds and rebinds the shared ``minion_instance.functions`` at its top via ``gen_modules()``; a sibling job's rebind landing between a job's retcode write and its read made ``_thread_return`` read a fresh, empty ``__context__`` and deliver ``EX_OK``. ``gen_modules()`` now returns the loader generation it built, and ``_thread_return``/``_thread_multi_return`` capture it and read the retcode from the loader the job owns instead of the shared attribute. The ``sys.reload_modules`` binding was pointed at a new ``_reload_modules`` wrapper so it keeps returning ``None`` on the wire. The proxy and delta-proxy job runners got the same loader-capture treatment.
26 changes: 17 additions & 9 deletions salt/metaproxy/deltaproxy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -658,7 +658,11 @@ def thread_return(cls, minion_instance, opts, data):
if f"{executor}.allow_missing_func" in minion_instance.executors
]
)
if function_name in minion_instance.functions or allow_missing_funcs is True:
# Own the loader for the length of this job so the func lookup, the retcode
# reset and the retcode read all resolve against the same generation, even if
# a concurrent sys.reload_modules rebinds minion_instance.functions. See #61830.
functions = minion_instance.functions
if function_name in functions or allow_missing_funcs is True:
try:
minion_blackout_violation = False
if minion_instance.connected and minion_instance.opts["pillar"].get(
Expand Down Expand Up @@ -690,15 +694,15 @@ def thread_return(cls, minion_instance, opts, data):
"saltutil.refresh_pillar allowed in blackout mode."
)

if function_name in minion_instance.functions:
func = minion_instance.functions[function_name]
if function_name in functions:
func = functions[function_name]
args, kwargs = salt.minion.load_args_and_kwargs(func, data["arg"], data)
else:
# only run if function_name is not in minion_instance.functions and allow_missing_funcs is True
func = function_name
args, kwargs = data["arg"], data
minion_instance.functions.pack["__context__"]["retcode"] = 0
minion_instance.functions.pack["__opts__"] = opts
functions.pack["__context__"]["retcode"] = 0
functions.pack["__opts__"] = opts
if isinstance(executors, str):
executors = [executors]
elif not isinstance(executors, list) or not executors:
Expand Down Expand Up @@ -740,7 +744,7 @@ def thread_return(cls, minion_instance, opts, data):
else:
ret["return"] = return_data

retcode = minion_instance.functions.pack["__context__"].get(
retcode = functions.pack["__context__"].get(
"retcode", salt.defaults.exitcodes.EX_OK
)
if retcode == salt.defaults.exitcodes.EX_OK:
Expand Down Expand Up @@ -900,6 +904,10 @@ def thread_multi_return(cls, minion_instance, opts, data):
else:
ret = {"return": {}, "retcode": {}, "success": {}}

# Own the loader for the length of this job so every func's lookup, retcode
# reset and retcode read resolve against the same generation, even if a
# concurrent sys.reload_modules rebinds minion_instance.functions. See #61830.
functions = minion_instance.functions
for ind in range(0, num_funcs):
if not multifunc_ordered:
ret["success"][data["fun"][ind]] = False
Expand Down Expand Up @@ -933,15 +941,15 @@ def thread_multi_return(cls, minion_instance, opts, data):
"saltutil.refresh_pillar allowed in blackout mode."
)

func = minion_instance.functions[data["fun"][ind]]
func = functions[data["fun"][ind]]

args, kwargs = salt.minion.load_args_and_kwargs(
func, data["arg"][ind], data
)
minion_instance.functions.pack["__context__"]["retcode"] = 0
functions.pack["__context__"]["retcode"] = 0
key = ind if multifunc_ordered else data["fun"][ind]
ret["return"][key] = func(*args, **kwargs)
retcode = minion_instance.functions.pack["__context__"].get("retcode", 0)
retcode = functions.pack["__context__"].get("retcode", 0)
if retcode == 0:
# No nonzero retcode in __context__ dunder. Check if return
# is a dictionary with a "result" or "success" key.
Expand Down
24 changes: 16 additions & 8 deletions salt/metaproxy/proxy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -422,7 +422,11 @@ def thread_return(cls, minion_instance, opts, data):
if f"{executor}.allow_missing_func" in minion_instance.executors
]
)
if function_name in minion_instance.functions or allow_missing_funcs is True:
# Own the loader for the length of this job so the func lookup, the retcode
# reset and the retcode read all resolve against the same generation, even if
# a concurrent sys.reload_modules rebinds minion_instance.functions. See #61830.
functions = minion_instance.functions
if function_name in functions or allow_missing_funcs is True:
try:
minion_blackout_violation = False
if minion_instance.connected and minion_instance.opts["pillar"].get(
Expand Down Expand Up @@ -454,14 +458,14 @@ def thread_return(cls, minion_instance, opts, data):
"saltutil.refresh_pillar allowed in blackout mode."
)

if function_name in minion_instance.functions:
func = minion_instance.functions[function_name]
if function_name in functions:
func = functions[function_name]
args, kwargs = salt.minion.load_args_and_kwargs(func, data["arg"], data)
else:
# only run if function_name is not in minion_instance.functions and allow_missing_funcs is True
func = function_name
args, kwargs = data["arg"], data
minion_instance.functions.pack["__context__"]["retcode"] = 0
functions.pack["__context__"]["retcode"] = 0
if isinstance(executors, str):
executors = [executors]
elif not isinstance(executors, list) or not executors:
Expand Down Expand Up @@ -501,7 +505,7 @@ def thread_return(cls, minion_instance, opts, data):
else:
ret["return"] = return_data

retcode = minion_instance.functions.pack["__context__"].get(
retcode = functions.pack["__context__"].get(
"retcode", salt.defaults.exitcodes.EX_OK
)
if retcode == salt.defaults.exitcodes.EX_OK:
Expand Down Expand Up @@ -655,6 +659,10 @@ def thread_multi_return(cls, minion_instance, opts, data):
else:
ret = {"return": {}, "retcode": {}, "success": {}}

# Own the loader for the length of this job so every func's lookup, retcode
# reset and retcode read resolve against the same generation, even if a
# concurrent sys.reload_modules rebinds minion_instance.functions. See #61830.
functions = minion_instance.functions
for ind in range(0, num_funcs):
if not multifunc_ordered:
ret["success"][data["fun"][ind]] = False
Expand Down Expand Up @@ -688,15 +696,15 @@ def thread_multi_return(cls, minion_instance, opts, data):
"saltutil.refresh_pillar allowed in blackout mode."
)

func = minion_instance.functions[data["fun"][ind]]
func = functions[data["fun"][ind]]

args, kwargs = salt.minion.load_args_and_kwargs(
func, data["arg"][ind], data
)
minion_instance.functions.pack["__context__"]["retcode"] = 0
functions.pack["__context__"]["retcode"] = 0
key = ind if multifunc_ordered else data["fun"][ind]
ret["return"][key] = func(*args, **kwargs)
retcode = minion_instance.functions.pack["__context__"].get("retcode", 0)
retcode = functions.pack["__context__"].get("retcode", 0)
if retcode == 0:
# No nonzero retcode in __context__ dunder. Check if return
# is a dictionary with a "result" or "success" key.
Expand Down
54 changes: 35 additions & 19 deletions salt/minion.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -565,15 +565,16 @@ def gen_modules(self, initial_load=False, context=None):
self.opts["resources"] = self._discover_resources()

self.utils = salt.loader.utils(self.opts, context=context)
self.functions = salt.loader.minion_mods(
functions = salt.loader.minion_mods(
self.opts, utils=self.utils, context=context
)
self.functions = functions
self.serializers = salt.loader.serializers(self.opts)
self.returners = salt.loader.returners(
self.opts, functions=self.functions, context=context
self.opts, functions=functions, context=context
)
self.proxy = salt.loader.proxy(
self.opts, functions=self.functions, returners=self.returners
self.opts, functions=functions, returners=self.returners
)
# Load resource connection modules (salt/resource/*.py) and build
# one execution-module loader per managed resource type.
Expand Down Expand Up @@ -632,20 +633,32 @@ def gen_modules(self, initial_load=False, context=None):
self.function_errors = {} # Keep the funcs clean
self.states = salt.loader.states(
self.opts,
functions=self.functions,
functions=functions,
utils=self.utils,
serializers=self.serializers,
context=context,
)
self.rend = salt.loader.render(
self.opts, functions=self.functions, context=context
)
self.rend = salt.loader.render(self.opts, functions=functions, context=context)
# self.matcher = Matcher(self.opts, self.functions)
self.matchers = salt.loader.matchers(self.opts)
self.functions["sys.reload_modules"] = self.gen_modules
functions["sys.reload_modules"] = self._reload_modules
self.executors = salt.loader.executors(
self.opts, functions=self.functions, proxy=self.proxy, context=context
self.opts, functions=functions, proxy=self.proxy, context=context
)
# Return the loader generation this call built so that a threaded job
# (multiprocessing=False) can own the exact loader it wrote its retcode
# into, rather than re-reading the shared self.functions attribute which
# a sibling job's gen_modules() may have rebound. See issue #61830.
return functions

def _reload_modules(self, initial_load=False, context=None):
"""
The ``sys.reload_modules`` execution function. gen_modules() returns the
rebuilt loader (a LazyLoader, which is not serializable), so it cannot be
bound to ``sys.reload_modules`` directly -- the returned value would be
put on the wire. This wrapper reloads the modules and returns None.
"""
self.gen_modules(initial_load=initial_load, context=context)

def _discover_resources(self):
"""
Expand Down Expand Up @@ -2778,9 +2791,14 @@ def _execute_job_function(
Executes a function within a job given it's name, the args and the executors.
It also checks if the function is allowed to run if 'blackout mode' is enabled.

``functions`` defaults to ``self.functions`` but callers may pass a
different loader (e.g. a per-resource-type loader) to route execution
to the correct module set.
``functions`` is the loader this job runs against. It defaults to
``self.functions`` for callers that do not thread a loader. Two callers
pass an explicit loader: a threaded job (multiprocessing: False) passes
the generation returned by gen_modules() so the func lookup and retcode
reset use the loader it wrote its retcode into rather than the shared
self.functions a sibling job may have rebound (#61830); a resource job
passes the per-resource-type loader to route execution to the correct
module set.
"""
if functions is None:
functions = self.functions
Expand Down Expand Up @@ -2870,7 +2888,7 @@ def _thread_return(cls, minion_instance, opts, data):
# covers; recorded in the finally below.
_exec_perf_start = time.perf_counter()

minion_instance.gen_modules()
functions = minion_instance.gen_modules()

fn_ = os.path.join(minion_instance.proc_dir, str(data["jid"]))

Expand Down Expand Up @@ -2952,7 +2970,7 @@ def _thread_return(cls, minion_instance, opts, data):
ret["retcode"] = salt.defaults.exitcodes.EX_OK
ret["success"] = True
else:
functions_to_use = minion_instance.functions
functions_to_use = functions
if (
ret.get("retcode") is None
and functions_to_use is not None
Expand Down Expand Up @@ -3381,7 +3399,7 @@ def _thread_multi_return(cls, minion_instance, opts, data):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)

minion_instance.gen_modules()
functions = minion_instance.gen_modules()

fn_ = os.path.join(minion_instance.proc_dir, str(data["jid"]))

Expand Down Expand Up @@ -3438,14 +3456,12 @@ def _thread_multi_return(cls, minion_instance, opts, data):
ret["success"][function_name] = False
try:
return_data = minion_instance._execute_job_function(
function_name, function_args, executors, opts, data
function_name, function_args, executors, opts, data, functions
)

key = ind if multifunc_ordered else data["fun"][ind]
ret["return"][key] = return_data
retcode = minion_instance.functions.pack["__context__"].get(
"retcode", 0
)
retcode = functions.pack["__context__"].get("retcode", 0)
if retcode == 0:
# No nonzero retcode in __context__ dunder. Check if return
# is a dictionary with a "result" or "success" key.
Expand Down
Loading
Loading