diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f762003b..24b970e63 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,10 +51,10 @@ if (${OS} STREQUAL "macos") message("Using HOMEBREW_PREFIX = ${HOMEBREW_PREFIX}") endif() -if (${OS} STREQUAL "macos") - find_package(Boost 1.90 REQUIRED CONFIG COMPONENTS filesystem) -else() +if (${OS} STREQUAL "linux" AND NOT DEFINED DAISY_FLATPAK) find_package(Boost 1.83 REQUIRED CONFIG COMPONENTS filesystem system) +else() + find_package(Boost 1.90 REQUIRED CONFIG COMPONENTS filesystem process) endif() message(STATUS "Boost version: ${Boost_VERSION}") diff --git a/Makefile b/Makefile index 5980fe705..a47a938b0 100644 --- a/Makefile +++ b/Makefile @@ -67,14 +67,14 @@ linux-coverage-build: ## Linux: Debian package .PHONY: debian debian: linux-build - cd ${linux_build_dir} && pack -G DEB + cd ${linux_build_dir} && cpack -G DEB ## Linux: Flatpak package .PHONY: flatpak flatpak: cd flatpak && \ flatpak-builder --force-clean --install-deps-from=flathub --repo=repo build dk.ku.daisy.yml && \ - flatpak build-bundle repo daisy-$(version).flatpak dk.ku.daisy + flatpak build-bundle repo daisy-$(daisy_version).flatpak dk.ku.daisy ## Linux test using standard build @@ -85,7 +85,7 @@ linux-test: $(linux_build_dir)/daisy cd $(linux_build_dir) && \ uv venv --allow-existing && \ uv pip install git+https://github.com/daisy-model/daisypy-test && \ - ctest -j 20 + ctest --output-on-failure ## Linux test using coverage build $(linux_coverage_build_dir)/daisy: linux-coverage-build diff --git a/cmake/Linux.cmake b/cmake/Linux.cmake index 0d5ee80ec..4c7ea11df 100644 --- a/cmake/Linux.cmake +++ b/cmake/Linux.cmake @@ -9,6 +9,7 @@ target_link_options(${DAISY_BIN_NAME} PRIVATE ${LINKER_OPTIONS}) target_link_libraries(${DAISY_BIN_NAME} PUBLIC cxsparse Boost::filesystem + $<$:Boost::process> ) install(TARGETS ${DAISY_BIN_NAME} RUNTIME DESTINATION bin) diff --git a/cmake/MacOS.cmake b/cmake/MacOS.cmake index ba1ec8180..8cb8bbe71 100644 --- a/cmake/MacOS.cmake +++ b/cmake/MacOS.cmake @@ -9,6 +9,7 @@ target_link_options(${DAISY_BIN_NAME} PRIVATE ${LINKER_OPTIONS}) target_link_libraries(${DAISY_BIN_NAME} PUBLIC cxsparse Boost::filesystem + Boost::process ) target_link_directories(${DAISY_BIN_NAME} PRIVATE ${EXTRA_SYSTEM_INCLUDE_DIRECTORIES}) diff --git a/cmake/MinGW.cmake b/cmake/MinGW.cmake index 584647c3c..6297b26ea 100644 --- a/cmake/MinGW.cmake +++ b/cmake/MinGW.cmake @@ -12,6 +12,7 @@ target_include_directories(${DAISY_CORE_NAME} PUBLIC include) target_link_libraries(${DAISY_CORE_NAME} PUBLIC cxsparse Boost::filesystem + Boost::process ) target_link_options(${DAISY_CORE_NAME} PRIVATE ${LINKER_OPTIONS}) diff --git a/flatpak/dk.ku.daisy.yml b/flatpak/dk.ku.daisy.yml index bd60f2fa7..da8bb611b 100644 --- a/flatpak/dk.ku.daisy.yml +++ b/flatpak/dk.ku.daisy.yml @@ -23,7 +23,7 @@ modules: - name: Boost buildsystem: simple build-commands: - - ./bootstrap.sh --prefix=/app --with-libraries=filesystem,system + - ./bootstrap.sh --prefix=/app --with-libraries=filesystem,process - ./b2 install link=static runtime-link=static cxxflags="-fPIC" -j $FLATPAK_BUILDER_N_JOBS cleanup: - '*' @@ -67,6 +67,7 @@ modules: config-opts: - --preset linux-gcc-portable - -DBoost_USE_STATIC_RUNTIME=ON + - -DDAISY_FLATPAK=ON sources: - type: git url: https://github.com/daisy-model/daisy.git diff --git a/include/util/run_cmd.h b/include/util/run_cmd.h deleted file mode 100644 index 9a02754e8..000000000 --- a/include/util/run_cmd.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef DAISY_UTIL_RUN_CMD_H -#define DAISY_UTIL_RUN_CMD_H -#include -#include - -std::string quote_if_unquoted(const std::string &); -std::pair run_cmd(const std::string & exe, const std::string & args, const std::string & name); - -#endif diff --git a/include/windows/windows_util.h b/include/windows/windows_util.h index dcc329176..bab979187 100644 --- a/include/windows/windows_util.h +++ b/include/windows/windows_util.h @@ -35,6 +35,5 @@ std::basic_string GetStringFromWindowsApi( TStringGetterFunc stringGetter std::basic_string get_exe_path(); std::basic_string get_daisy_home_from_exe_path(); -//std::basic_string run_cmd(std::basic_string exe, std::basic_string args, std::basic_string name); #endif -#endif \ No newline at end of file +#endif diff --git a/src/programs/program_spawn.C b/src/programs/program_spawn.C index 9f074d6eb..6c00ae8e2 100644 --- a/src/programs/program_spawn.C +++ b/src/programs/program_spawn.C @@ -30,6 +30,8 @@ #include #include +#include +#include #include #include @@ -41,8 +43,6 @@ #include #include -#include "util/run_cmd.h" - namespace { void handle_success(const std::string &name, Treelog &msg) { std::ostringstream tmp; @@ -64,6 +64,12 @@ namespace { file << "Exit code " << status; msg.message(tmp.str()); } + + struct Job { + std::string name; + std::vector args; + }; + } struct ProgramSpawn : public Program { @@ -77,8 +83,7 @@ struct ProgramSpawn : public Program { const int length; // State. - std::vector cmds; - std::vector names; + std::list jobs; void prepare_cmds(Treelog& msg) { for (int i = 0; i < length; ++i) { @@ -117,55 +122,65 @@ struct ProgramSpawn : public Program { msg.message (tmp.str ()); return; } - std::string cmd = " -d " + quote_if_unquoted(directory_one.name()) - + " -D " + quote_if_unquoted(input_directory.name()) - + " -q " + quote_if_unquoted(file_one.name()); + std::vector args{ + "-d", directory_one.name(), + "-D", input_directory.name(), + "-q", + file_one.name()}; if (program_one != Attribute::None()) { - cmd += " -p " + quote_if_unquoted(program_one.name()); + args.push_back("-p"); + args.push_back(program_one.name()); } - cmds.push_back(cmd); - names.push_back(directory_one.name()); + jobs.push_back({directory_one.name(), args}); } void run_cmds(Treelog& msg) { using namespace std::chrono_literals; + namespace bp = boost::process::v2; + namespace ba = boost::asio; + + ba::thread_pool pool; + auto exec = pool.get_executor(); + ba::strand strand(exec); + int running = 0; - std::list>> progs; - for (int idx = 0; idx < cmds.size(); ++idx) { - // If parallel > 0, then we only start that many tasks simultaneously - while (parallel > 0 && running >= parallel) { - for (auto it = progs.begin(); it != progs.end();) { - if (it->wait_for(0.1s) == std::future_status::ready) { - auto [status, name] = it->get(); - if (status == 0) { - handle_success(name, msg); - } - else { - handle_failure(status, name, msg); - } - it = progs.erase(it); + std::function start_next; + + // Function that recursively starts processes + start_next = [&]() { + ba::dispatch(strand, [&]() { + if ((parallel > 0 && running >= parallel) || jobs.empty()) { + // We are at the limit or done + return; + } + + auto job = jobs.front(); + jobs.pop_front(); + auto name = job.name; + msg.message("Running " + name); + ++running; + + auto proc = std::make_shared(exec, exe.name(), job.args); + + // Capturing proc by value keeps the pointer alive until we are done + proc->async_wait([&, proc, name](boost::system::error_code ec, int exit_code) { + ba::dispatch(strand, [&]() { --running; - } else { - ++it; + // Try to start a new one + start_next(); + }); + if (!ec && exit_code == 0) { + handle_success(name, msg); } - } - } - msg.message("Running " + names[idx]); - progs.push_back(std::async(std::launch::async, run_cmd, exe.name(), cmds[idx], names[idx])); - ++running; - } - // Wait for all tasks to finish - for (auto it = progs.begin(); it != progs.end();) { - it->wait(); - auto [status, name] = it->get(); - if (status == 0) { - handle_success(name, msg); - } - else { - handle_failure(status, name, msg); - } - it = progs.erase(it); - } + else { + handle_failure(exit_code, name, msg); + } + }); + start_next(); // Recursively start more until we hit the limit or are done + }); + }; + start_next(); + pool.join(); // Wait until everything is done. } // Use. @@ -229,8 +244,7 @@ struct ProgramSpawn : public Program { length (std::max ({ program.size (), directory.size (), file.size ()})), - cmds(), - names() + jobs() { } ~ProgramSpawn () { } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 463a027ab..b6040bd5c 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -18,7 +18,6 @@ target_sources(${DAISY_CORE_NAME} PRIVATE nrutil.C path.C point.C - run_cmd.C scope.C scope_exchange.C scope_id.C diff --git a/src/util/run_cmd.C b/src/util/run_cmd.C deleted file mode 100644 index b3bf7d3a2..000000000 --- a/src/util/run_cmd.C +++ /dev/null @@ -1,54 +0,0 @@ -#include "util/run_cmd.h" -#include -#include - -std::string quote_if_unquoted(const std::string& str) { - if (str.size() < 2 || str.front() != '"' || str.back() != '"') { - return '"' + str + '"'; - } - return str; -} - -#if defined(_WIN32) || defined(__CYGWIN__32__) -#include -#include -std::pair run_cmd(const std::string & exe, const std::string & args, const std::string & name) { - STARTUPINFO si; - PROCESS_INFORMATION pi; - ZeroMemory( &si, sizeof(si) ); - si.cb = sizeof(si); - ZeroMemory( &pi, sizeof(pi) ); - int status = 0; - if (CreateProcessA(exe.c_str(), const_cast(args.c_str()), NULL, NULL, false, 0, NULL, NULL, &si, &pi)) { - // WaitForSingleObject returns WAIT_OBJECT_0 on success - // https://learn.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-waitforsingleobject - if (WaitForSingleObject(pi.hProcess, INFINITE) == 0) { - // Check the exit code. Is zero if there was an error - // https://learn.microsoft.com/en-gb/windows/win32/api/processthreadsapi/nf-processthreadsapi-getexitcodeprocess - DWORD exit_code; - if (GetExitCodeProcess(pi.hProcess, &exit_code)) { - status = exit_code; - } - else { - status = -3; - } - } - else { - status = -2; - } - } - else { - status = -1; - } - CloseHandle(pi.hProcess); - CloseHandle(pi.hThread); - return std::make_pair(status, name); - -} -#else -#include -std::pair run_cmd(const std::string & exe, const std::string & args, const std::string & name) { - int status = std::system((quote_if_unquoted(exe) + " " + args).c_str()); - return std::make_pair(status, name); -} -#endif