Fix spawn macos

CMakeLists.txt

@@ -51,10 +51,10 @@ if (${OS} STREQUAL "macos")
   message("Using HOMEBREW_PREFIX = ${HOMEBREW_PREFIX}")
 endif()
 
-if (${OS} STREQUAL "macos")
-  find_package(Boost 1.90 REQUIRED CONFIG COMPONENTS filesystem)
-else()
+if (${OS} STREQUAL "linux" AND NOT DEFINED DAISY_FLATPAK)
   find_package(Boost 1.83 REQUIRED CONFIG COMPONENTS filesystem system)
+else()
+  find_package(Boost 1.90 REQUIRED CONFIG COMPONENTS filesystem process)
 endif()
 message(STATUS "Boost version: ${Boost_VERSION}")
 

Makefile

@@ -67,14 +67,14 @@ linux-coverage-build:
 ## Linux: Debian package
 .PHONY: debian
 debian: linux-build
-	cd ${linux_build_dir} && pack -G DEB
+	cd ${linux_build_dir} && cpack -G DEB
 
 ## Linux: Flatpak package
 .PHONY: flatpak
 flatpak:
 	cd flatpak && \
 	flatpak-builder --force-clean --install-deps-from=flathub --repo=repo build dk.ku.daisy.yml && \
-	flatpak build-bundle repo daisy-$(version).flatpak dk.ku.daisy
+	flatpak build-bundle repo daisy-$(daisy_version).flatpak dk.ku.daisy
 
 
 ## Linux test using standard build
@@ -85,7 +85,7 @@ linux-test: $(linux_build_dir)/daisy
 	cd $(linux_build_dir) && \
 	uv venv --allow-existing && \
 	uv pip install git+https://github.com/daisy-model/daisypy-test && \
-	ctest -j 20
+	ctest --output-on-failure
 
 ## Linux test using coverage build
 $(linux_coverage_build_dir)/daisy: linux-coverage-build

cmake/Linux.cmake

@@ -9,6 +9,7 @@ target_link_options(${DAISY_BIN_NAME} PRIVATE ${LINKER_OPTIONS})
 target_link_libraries(${DAISY_BIN_NAME} PUBLIC
   cxsparse
   Boost::filesystem
+  $<$<BOOL:${DAISY_FLATPAK}>:Boost::process>
 )
 
 install(TARGETS ${DAISY_BIN_NAME} RUNTIME DESTINATION bin)

cmake/MacOS.cmake

@@ -9,6 +9,7 @@ target_link_options(${DAISY_BIN_NAME} PRIVATE ${LINKER_OPTIONS})
 target_link_libraries(${DAISY_BIN_NAME} PUBLIC
   cxsparse
   Boost::filesystem
+  Boost::process
 )
 target_link_directories(${DAISY_BIN_NAME} PRIVATE ${EXTRA_SYSTEM_INCLUDE_DIRECTORIES})
 

cmake/MinGW.cmake

@@ -12,6 +12,7 @@ target_include_directories(${DAISY_CORE_NAME} PUBLIC include)
 target_link_libraries(${DAISY_CORE_NAME} PUBLIC
   cxsparse
   Boost::filesystem
+  Boost::process
 )
 target_link_options(${DAISY_CORE_NAME} PRIVATE ${LINKER_OPTIONS})
 

flatpak/dk.ku.daisy.yml

@@ -23,7 +23,7 @@ modules:
   - name: Boost
     buildsystem: simple
     build-commands:
-      - ./bootstrap.sh --prefix=/app --with-libraries=filesystem,system
+      - ./bootstrap.sh --prefix=/app --with-libraries=filesystem,process
       - ./b2 install link=static runtime-link=static cxxflags="-fPIC" -j $FLATPAK_BUILDER_N_JOBS
     cleanup:
       - '*'
@@ -67,6 +67,7 @@ modules:
     config-opts:
       - --preset linux-gcc-portable
       - -DBoost_USE_STATIC_RUNTIME=ON
+      - -DDAISY_FLATPAK=ON
     sources:
       - type: git
         url: https://github.com/daisy-model/daisy.git

include/windows/windows_util.h

@@ -35,6 +35,5 @@ std::basic_string<TChar> GetStringFromWindowsApi( TStringGetterFunc stringGetter
 
 std::basic_string<TCHAR> get_exe_path();
 std::basic_string<TCHAR> get_daisy_home_from_exe_path();
-//std::basic_string<TCHAR> run_cmd(std::basic_string<TCHAR> exe, std::basic_string<TCHAR> args, std::basic_string<TCHAR> name);
 #endif
-#endif
+#endif

src/programs/program_spawn.C

@@ -30,6 +30,8 @@
 
 #include <boost/filesystem.hpp>
 #include <boost/dll.hpp>
+#include <boost/asio.hpp>
+#include <boost/process/v2/process.hpp>
 
 #include <chrono>
 #include <future>
@@ -41,8 +43,6 @@
 #include <fstream>
 #include <utility>
 
-#include "util/run_cmd.h"
-
 namespace {
   void handle_success(const std::string &name, Treelog &msg) {
     std::ostringstream tmp;
@@ -64,6 +64,12 @@ namespace {
     file << "Exit code " << status;
     msg.message(tmp.str());
   }
+
+  struct Job {
+    std::string name;
+    std::vector<std::string> args;
+  };
+
 }
 
 struct ProgramSpawn : public Program {
@@ -77,8 +83,7 @@ struct ProgramSpawn : public Program {
   const int length;
 
   // State.
-  std::vector<std::string> cmds;
-  std::vector<std::string> names;
+  std::list<Job> jobs;
 
   void prepare_cmds(Treelog& msg) {
     for (int i = 0; i < length; ++i) {
@@ -117,55 +122,65 @@ struct ProgramSpawn : public Program {
 	    msg.message (tmp.str ());
 	    return;
     }
-	  std::string cmd = " -d " + quote_if_unquoted(directory_one.name())
-                    + " -D " + quote_if_unquoted(input_directory.name())
-			              + " -q " + quote_if_unquoted(file_one.name());
+    std::vector<std::string> args{
+      "-d", directory_one.name(),
+      "-D", input_directory.name(),
+      "-q",
+      file_one.name()};
     if (program_one != Attribute::None()) {
-      cmd += " -p " + quote_if_unquoted(program_one.name());
+      args.push_back("-p");
+      args.push_back(program_one.name());
     }
-    cmds.push_back(cmd);
-    names.push_back(directory_one.name());
+    jobs.push_back({directory_one.name(), args});
   }
 
   void run_cmds(Treelog& msg) {
     using namespace std::chrono_literals;
+    namespace bp = boost::process::v2;
+    namespace ba = boost::asio;
+
+    ba::thread_pool pool;
+    auto exec = pool.get_executor();
+    ba::strand<decltype(exec)> strand(exec);
+
     int running = 0;
-    std::list<std::future<std::pair<int, std::string>>> progs;
-    for (int idx = 0; idx < cmds.size(); ++idx) {
-      // If parallel > 0, then we only start that many tasks simultaneously
-      while (parallel > 0  && running >= parallel) {
-        for (auto it = progs.begin(); it != progs.end();) {
-          if (it->wait_for(0.1s) == std::future_status::ready) {
-            auto [status, name] = it->get();
-            if (status == 0) {
-              handle_success(name, msg);
-            }
-            else {
-              handle_failure(status, name, msg);
-            }
-            it = progs.erase(it);
+    std::function<void()> start_next;
+
+    // Function that recursively starts processes
+    start_next = [&]() {
+      ba::dispatch(strand, [&]() {
+        if ((parallel > 0 && running >= parallel) || jobs.empty()) {
+          // We are at the limit or done
+          return;
+        }
+
+        auto job = jobs.front();
+        jobs.pop_front();
+        auto name = job.name;
+        msg.message("Running " + name);
+        ++running;
+
+        auto proc = std::make_shared<bp::process>(exec, exe.name(), job.args);
+
+        // Capturing proc by value keeps the pointer alive until we are done
+        proc->async_wait([&, proc, name](boost::system::error_code ec, int exit_code) {
+          ba::dispatch(strand, [&]() {
             --running;
-          } else {
-            ++it;
+            // Try to start a new one
+            start_next();
+          });
+          if (!ec && exit_code == 0) {
+            handle_success(name, msg);
           }
-        }
-      }
-      msg.message("Running " + names[idx]);
-      progs.push_back(std::async(std::launch::async, run_cmd, exe.name(), cmds[idx], names[idx]));
-      ++running;
-    }
-    // Wait for all tasks to finish
-    for (auto it = progs.begin(); it != progs.end();) {
-      it->wait();
-      auto [status, name] = it->get();
-      if (status == 0) {
-        handle_success(name, msg);
-      }
-      else {
-        handle_failure(status, name, msg);
-      }
-      it = progs.erase(it);
-    }
+          else {
+            handle_failure(exit_code, name, msg);
+          }
+        });
+        start_next(); // Recursively start more until we hit the limit or are done
+      });
+    };
+    start_next();
+    pool.join(); // Wait until everything is done.
   }
 
   // Use.
@@ -229,8 +244,7 @@ struct ProgramSpawn : public Program {
       length (std::max ({ program.size (),
 			  directory.size (),
 			  file.size ()})),
-      cmds(),
-      names()
+      jobs()
   { }
   ~ProgramSpawn ()
   {  }

src/util/CMakeLists.txt

@@ -18,7 +18,6 @@ target_sources(${DAISY_CORE_NAME} PRIVATE
   nrutil.C
   path.C
   point.C
-  run_cmd.C
   scope.C
   scope_exchange.C
   scope_id.C