From 8f54e7dd2391832bafc638705effb29e125f8759 Mon Sep 17 00:00:00 2001 From: noah-weingarden Date: Fri, 18 Apr 2025 00:01:49 -0400 Subject: [PATCH 1/3] Give partitioner an arg for is_executable() --- madoop/mapreduce.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/madoop/mapreduce.py b/madoop/mapreduce.py index 7cacde5..2ed7c1f 100644 --- a/madoop/mapreduce.py +++ b/madoop/mapreduce.py @@ -44,7 +44,7 @@ def mapreduce( is_executable(map_exe) is_executable(reduce_exe) if partitioner: - is_executable(partitioner) + is_executable(partitioner, "2") # Create a tmp directory which will be automatically cleaned up with tempfile.TemporaryDirectory(prefix="madoop-") as tmpdir: @@ -157,7 +157,7 @@ def normalize_input_paths(input_path): return input_paths -def is_executable(exe): +def is_executable(exe, *args): """Verify exe is executable and raise exception if it is not. Execute exe with an empty string input and verify that it returns zero. We @@ -168,7 +168,7 @@ def is_executable(exe): exe = pathlib.Path(exe).resolve() try: subprocess.run( - str(exe), + [str(exe), *args], shell=False, input="".encode(), stdout=subprocess.PIPE, From ababb64d3cae8dad81defbc44e39d36d6ce50838 Mon Sep 17 00:00:00 2001 From: noah-weingarden <33741795+noah-weingarden@users.noreply.github.com> Date: Thu, 24 Apr 2025 17:13:32 -0400 Subject: [PATCH 2/3] Add comment, pass actual num_reducers --- madoop/mapreduce.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/madoop/mapreduce.py b/madoop/mapreduce.py index 2ed7c1f..e4c4a88 100644 --- a/madoop/mapreduce.py +++ b/madoop/mapreduce.py @@ -43,8 +43,9 @@ def mapreduce( # Executable scripts must have valid shebangs is_executable(map_exe) is_executable(reduce_exe) + # The partitioner executable expects to receive num_reducers as an arg if partitioner: - is_executable(partitioner, "2") + is_executable(partitioner, str(num_reducers)) # Create a tmp directory which will be automatically cleaned up with tempfile.TemporaryDirectory(prefix="madoop-") as tmpdir: From da3a0ac34a39e5b8a393c2961ca183d750a93072 Mon Sep 17 00:00:00 2001 From: noah-weingarden Date: Mon, 28 Apr 2025 13:55:00 -0400 Subject: [PATCH 3/3] Empty commit because CI didn't run for some reason