From 6ed12159ff6b4fb6bf59c8f0f911d8201c8c1a42 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 16 Jul 2026 11:57:39 -0400 Subject: [PATCH 1/6] Reduce overhead of copying std::map multiple times --- include/core/Layer.hpp | 2 +- include/netcdf/NetCDFManager.hpp | 2 +- src/core/Layer.cpp | 2 +- src/core/NgenSimulation.cpp | 4 ++-- src/netcdf/NetCDFManager.cpp | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/include/core/Layer.hpp b/include/core/Layer.hpp index 308fd61b4b..da81591589 100644 --- a/include/core/Layer.hpp +++ b/include/core/Layer.hpp @@ -132,7 +132,7 @@ namespace ngen std::string unit_name() const; virtual std::vector required_checkpoint_units() const; - virtual std::map get_catchment_output_data_for_timestep(); + virtual const std::map& get_catchment_output_data_for_timestep(); virtual void set_simulations_output_format(std::vector out_formats); virtual std::vector get_simulations_output_format(); protected: diff --git a/include/netcdf/NetCDFManager.hpp b/include/netcdf/NetCDFManager.hpp index bb3c97f09d..482ca85692 100644 --- a/include/netcdf/NetCDFManager.hpp +++ b/include/netcdf/NetCDFManager.hpp @@ -70,7 +70,7 @@ class NetCDFManager void add_output_variable_data_from_formulation(); // Add catchment output data to the file (for writing) - void write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values); + void write_simulations_response_from_formulation(size_t time_index, const std::map& catchment_output_values); void primary_netcdf_writer(size_t time_index, const std::map& catchment_output_values); void secondary_netcdf_worker(const std::map& catchment_output_values); diff --git a/src/core/Layer.cpp b/src/core/Layer.cpp index 2c318a2590..6a51b526da 100644 --- a/src/core/Layer.cpp +++ b/src/core/Layer.cpp @@ -188,7 +188,7 @@ void ngen::Layer::load_hot_start(std::shared_ptr snapshot } } -std::map ngen::Layer::get_catchment_output_data_for_timestep(){ +const std::map& ngen::Layer::get_catchment_output_data_for_timestep(){ return catchment_output_values; } diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index c3dfce62d7..e01df9d04a 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -220,8 +220,8 @@ void NgenSimulation::advance_models_one_output_step() std::vector output_formats = layer->get_simulations_output_format(); if (std::find(output_formats.begin(), output_formats.end(), "netcdf") != output_formats.end()){ produce_netcdf_format_ = true; - std::map catchment_output_vals = layer->get_catchment_output_data_for_timestep(); - nc_manager_->write_simulations_response_from_formulation(simulation_step_,catchment_output_vals); + const std::map &catchment_output_vals = layer->get_catchment_output_data_for_timestep(); + nc_manager_->write_simulations_response_from_formulation(simulation_step_, catchment_output_vals); } #endif //NGEN_WITH_NETCDF diff --git a/src/netcdf/NetCDFManager.cpp b/src/netcdf/NetCDFManager.cpp index 0036190c51..93e51259e2 100644 --- a/src/netcdf/NetCDFManager.cpp +++ b/src/netcdf/NetCDFManager.cpp @@ -240,7 +240,7 @@ static std::vector string_split(std::string str, char delimiter) return res; } -void NetCDFManager::write_simulations_response_from_formulation(size_t time_index, std::map catchment_output_values) +void NetCDFManager::write_simulations_response_from_formulation(size_t time_index, const std::map& catchment_output_values) { std::map output_values; try{ From 9af6ba371784a826671a285228ef87f4cf7e7610 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Thu, 16 Jul 2026 14:54:34 -0400 Subject: [PATCH 2/6] Implement restart of the netCDF file on checkpoint restart --- include/core/NgenSimulation.hpp | 2 +- include/netcdf/NetCDFFile.hpp | 9 ++++++++- include/netcdf/NetCDFManager.hpp | 9 ++++----- src/NGen.cpp | 16 +++++++++++----- src/core/NgenSimulation.cpp | 4 ++-- src/netcdf/NetCDFFile.cpp | 28 ++++++++++++++++++---------- src/netcdf/NetCDFManager.cpp | 19 +++++++++---------- 7 files changed, 53 insertions(+), 34 deletions(-) diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index fde8f94c51..f011a52acf 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -101,7 +101,7 @@ class NgenSimulation * @param merge_all_ranks Whether to also get the units from other MPI ranks. This should only be `true` when calling blocking MPI processes is safe for the program. */ std::vector required_checkpoint_units(bool merge_all_ranks) const; - void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name); + void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, NetCDFOpenMode open_mode); private: void advance_models_one_output_step(); diff --git a/include/netcdf/NetCDFFile.hpp b/include/netcdf/NetCDFFile.hpp index 97093fbe00..2a3ad36814 100644 --- a/include/netcdf/NetCDFFile.hpp +++ b/include/netcdf/NetCDFFile.hpp @@ -16,10 +16,17 @@ #endif #include "NetCDFVar.hpp" + +enum class NetCDFOpenMode { + OPEN_READ, + OPEN_WRITE, + CREATE +}; + class NetCDFFile { public: - NetCDFFile(const std::string& filename, bool write_only, bool is_mpi); + NetCDFFile(const std::string& filename, NetCDFOpenMode open_mode, bool is_mpi); ~NetCDFFile(); void load_attributes(std::shared_ptr nc_var) ; diff --git a/include/netcdf/NetCDFManager.hpp b/include/netcdf/NetCDFManager.hpp index 482ca85692..75aec142a0 100644 --- a/include/netcdf/NetCDFManager.hpp +++ b/include/netcdf/NetCDFManager.hpp @@ -28,7 +28,7 @@ class NetCDFManager { public: NetCDFManager(std::shared_ptr manager, - const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs); + const std::string& output_name, Simulation_Time const& sim_time, NetCDFOpenMode open_mode, int mpi_rank, int mpi_num_procs); // Constructor for read-only NetCDF (no MPI needed) NetCDFManager(const std::string& filename, bool read_only); @@ -43,9 +43,6 @@ class NetCDFManager void gather_all_catchments(const std::vector& catchments_in_proc); - //set up netcdf dimensions and variables - void define_catchment_netcdf_components(); - // List variable names std::vector list_variables() const; @@ -77,12 +74,14 @@ class NetCDFManager ~NetCDFManager(); private: + /* Set up netcdf dimensions and variables. + Note: A copy of Simulation_Time is passed because the object is expected to be modified and discarded at the end of the function. */ + void define_catchment_netcdf_components(Simulation_Time sim_time); bool read_only_; std::string nc_filename_; std::unique_ptr nc_file_; std::vector vars_; std::shared_ptr manager_; - std::shared_ptr sim_time_; size_t num_timesteps_; int num_catchments_ = 0; std::vector catchments_; diff --git a/src/NGen.cpp b/src/NGen.cpp index 71de0b6f27..3c39598dab 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -714,11 +714,6 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { std::move(nexus_indexes), mpi_rank, mpi_num_procs); - #if NGEN_WITH_NETCDF - if (std::find(output_formats.begin(), output_formats.end(), "netcdf") != output_formats.end()){ - simulation->create_netcdf_writer(manager, "catchment_output"); - } - #endif //NGEN_WITH_NETCDF auto time_done_init = std::chrono::steady_clock::now(); std::chrono::duration time_elapsed_init = time_done_init - time_start; LOG("[TIMING]: Init: " + std::to_string(time_elapsed_init.count()), LogLevel::INFO); @@ -732,6 +727,9 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { } } +#if NGEN_WITH_NETCDF + NetCDFOpenMode netcdf_open_mode = NetCDFOpenMode::CREATE; +#endif // NGEN_WITH_NETCDF { // optionally load a checkpoint if configured auto checkpoint_loader = state_saving_config.checkpoint_loader(); if (checkpoint_loader) { @@ -740,8 +738,16 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { std::shared_ptr snapshot_loader = checkpoint_loader->initialize_checkpoint_snapshot(required_units); simulation->load_checkpoint(snapshot_loader); +#if NGEN_WITH_NETCDF + netcdf_open_mode = NetCDFOpenMode::OPEN_WRITE; +#endif //NGEN_WITH_NETCDF } } +#if NGEN_WITH_NETCDF + if (std::find(output_formats.begin(), output_formats.end(), "netcdf") != output_formats.end()) { + simulation->create_netcdf_writer(manager, "catchment_output", netcdf_open_mode); + } +#endif //NGEN_WITH_NETCDF if (state_saving_config.has_checkpoint_saver()) { int checkpoint_frequency; diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index e01df9d04a..c18a03cd06 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -524,10 +524,10 @@ void NgenSimulation::serialize(Archive& ar, const unsigned int version) { #endif //NGEN_WITH_NEXUSES } -void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name) +void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, NetCDFOpenMode open_mode) { #if NGEN_WITH_NETCDF - this->nc_manager_ = std::make_unique(manager, nc_output_file_name, *sim_time_, mpi_rank_, mpi_num_procs_); + this->nc_manager_ = std::make_unique(manager, nc_output_file_name, *sim_time_, open_mode, mpi_rank_, mpi_num_procs_); #endif } diff --git a/src/netcdf/NetCDFFile.cpp b/src/netcdf/NetCDFFile.cpp index 996a3d3ac1..5e2252c2b9 100644 --- a/src/netcdf/NetCDFFile.cpp +++ b/src/netcdf/NetCDFFile.cpp @@ -23,19 +23,27 @@ } while (0) -NetCDFFile::NetCDFFile(const std::string& filename, bool write_only, bool is_mpi) +NetCDFFile::NetCDFFile(const std::string& filename, NetCDFOpenMode open_mode, bool is_mpi) : nc_file_name_(filename), is_mpi_(is_mpi) { - int mode = NC_NETCDF4; - if(write_only){ - read_only_ = false; - NC_CHECK(nc_create(nc_file_name_.c_str(), NC_NETCDF4 | NC_CLOBBER, &ncid_), "Creating NetCDF file failed"); + switch (open_mode) { + case NetCDFOpenMode::OPEN_READ: + read_only_ = true; + NC_CHECK(nc_open(nc_file_name_.c_str(), NC_NOWRITE, &ncid_), "Opening NetCDF file in read-only mode failed."); + break; + case NetCDFOpenMode::OPEN_WRITE: + read_only_ = false; + NC_CHECK(nc_open(nc_file_name_.c_str(), NC_WRITE, &ncid_), "Opening NetCDF file in write mode failed."); + break; + case NetCDFOpenMode::CREATE: + read_only_ = false; + NC_CHECK(nc_create(nc_file_name_.c_str(), NC_NETCDF4 | NC_CLOBBER, &ncid_), "Creating NetCDF file failed"); + break; + default: + std::string err_msg = "Invalid open mode for NetCDFFile"; + LOG(LogLevel::FATAL, err_msg); + throw std::runtime_error(err_msg); } - else{ - read_only_ = true; - NC_CHECK(nc_open(nc_file_name_.c_str(), NC_NOWRITE, &ncid_), "Opening NetCDF file failed"); - } -// } if(read_only_){ load_variables(); //load all netcdf data to objects. } diff --git a/src/netcdf/NetCDFManager.cpp b/src/netcdf/NetCDFManager.cpp index 93e51259e2..fe91b56000 100644 --- a/src/netcdf/NetCDFManager.cpp +++ b/src/netcdf/NetCDFManager.cpp @@ -10,10 +10,9 @@ #include NetCDFManager::NetCDFManager(std::shared_ptr manager, - const std::string& output_name, Simulation_Time const& sim_time, int mpi_rank, int mpi_num_procs) + const std::string& output_name, Simulation_Time const& sim_time, NetCDFOpenMode open_mode, int mpi_rank, int mpi_num_procs) + : manager_{manager} { - manager_ = manager; - sim_time_ = std::make_shared(sim_time); std::filesystem::path check_path(output_name); if(check_path.is_absolute()){ nc_filename_ = output_name; @@ -47,8 +46,8 @@ NetCDFManager::NetCDFManager(std::shared_ptr m } gather_all_catchments(catchments_in_proc); if (rank_ == 0){ - nc_file_ = std::make_unique(nc_filename_, true, is_mpi_); - define_catchment_netcdf_components(); + nc_file_ = std::make_unique(nc_filename_, open_mode, is_mpi_); + define_catchment_netcdf_components(sim_time); } #if NGEN_WITH_MPI if (comm_ != MPI_COMM_NULL) //This check is important if the user runs MPI with a single process. @@ -130,7 +129,7 @@ void NetCDFManager::gather_all_catchments(const std::vector& catchments #endif } -void NetCDFManager::define_catchment_netcdf_components() +void NetCDFManager::define_catchment_netcdf_components(Simulation_Time sim_time) { std::string name; int dim_id; @@ -141,7 +140,7 @@ void NetCDFManager::define_catchment_netcdf_components() try { name = "time"; - int num_timesteps = sim_time_->get_total_output_times(); + int num_timesteps = sim_time.get_total_output_times(); dim_id = add_dimension(name, num_timesteps); dim_ids = {dim_id}; names = {name}; @@ -149,11 +148,11 @@ void NetCDFManager::define_catchment_netcdf_components() //add timestep values and attributes for time std::vector time_epoch_seconds(num_timesteps); - time_epoch_seconds[0] = sim_time_->get_current_epoch_time(); + time_epoch_seconds[0] = sim_time.get_current_epoch_time(); for(int time_index = 1; time_index < num_timesteps; time_index++) { - sim_time_->advance_timestep(); - time_epoch_seconds[time_index] = sim_time_->get_current_epoch_time(); + sim_time.advance_timestep(); + time_epoch_seconds[time_index] = sim_time.get_current_epoch_time(); } nc_file_->write_variable_data(name, time_epoch_seconds); nc_file_->write_attribute_to_ncvar(name, "units", "Seconds since 1970-01-01 00:00:00"); From 81e5fb2d9d1209c2063d830b678325893b47f809 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 17 Jul 2026 09:01:42 -0400 Subject: [PATCH 3/6] Switch NetCDFFIle to use NetCDFOpenMode --- include/netcdf/NetCDFManager.hpp | 4 ++-- src/forcing/NetCDFPerFeatureDataProvider.cpp | 2 +- src/netcdf/NetCDFManager.cpp | 16 ++++++++-------- test/core/NetCDFCreatorTest.cpp | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/include/netcdf/NetCDFManager.hpp b/include/netcdf/NetCDFManager.hpp index 75aec142a0..d8e3ad1bef 100644 --- a/include/netcdf/NetCDFManager.hpp +++ b/include/netcdf/NetCDFManager.hpp @@ -31,7 +31,7 @@ class NetCDFManager const std::string& output_name, Simulation_Time const& sim_time, NetCDFOpenMode open_mode, int mpi_rank, int mpi_num_procs); // Constructor for read-only NetCDF (no MPI needed) - NetCDFManager(const std::string& filename, bool read_only); + NetCDFManager(const std::string& filename, NetCDFOpenMode open_mode); // Default constructor for mdframe tests NetCDFManager(); @@ -77,7 +77,7 @@ class NetCDFManager /* Set up netcdf dimensions and variables. Note: A copy of Simulation_Time is passed because the object is expected to be modified and discarded at the end of the function. */ void define_catchment_netcdf_components(Simulation_Time sim_time); - bool read_only_; + NetCDFOpenMode open_mode_; std::string nc_filename_; std::unique_ptr nc_file_; std::vector vars_; diff --git a/src/forcing/NetCDFPerFeatureDataProvider.cpp b/src/forcing/NetCDFPerFeatureDataProvider.cpp index bf225277c2..5b5d5d9742 100644 --- a/src/forcing/NetCDFPerFeatureDataProvider.cpp +++ b/src/forcing/NetCDFPerFeatureDataProvider.cpp @@ -53,7 +53,7 @@ NetCDFPerFeatureDataProvider::NetCDFPerFeatureDataProvider(std::string input_pat //nc_set_chunk_cache(sizep, nelemsp, preemptionp); //open the file - nc_manager = std::make_shared(input_path, true); + nc_manager = std::make_shared(input_path, NetCDFOpenMode::OPEN_READ); nc_manager->open_file(); //nc_get_chunk_cache(&sizep, &nelemsp, &preemptionp); diff --git a/src/netcdf/NetCDFManager.cpp b/src/netcdf/NetCDFManager.cpp index fe91b56000..b0893bfe0e 100644 --- a/src/netcdf/NetCDFManager.cpp +++ b/src/netcdf/NetCDFManager.cpp @@ -11,7 +11,7 @@ NetCDFManager::NetCDFManager(std::shared_ptr manager, const std::string& output_name, Simulation_Time const& sim_time, NetCDFOpenMode open_mode, int mpi_rank, int mpi_num_procs) - : manager_{manager} + : manager_{manager}, open_mode_{open_mode} { std::filesystem::path check_path(output_name); if(check_path.is_absolute()){ @@ -55,14 +55,14 @@ NetCDFManager::NetCDFManager(std::shared_ptr m #endif } -NetCDFManager::NetCDFManager(const std::string& filename, bool read_only) - : read_only_(true) +NetCDFManager::NetCDFManager(const std::string& filename, NetCDFOpenMode open_mode) + : open_mode_{open_mode} { - if (read_only_){ - nc_file_ = std::make_unique(filename, !read_only, false); + if (open_mode_ == NetCDFOpenMode::OPEN_READ) { + nc_file_ = std::make_unique(filename, open_mode, false); } else{ - throw std::runtime_error("Write only non-MPI function not implemented."); + throw std::runtime_error("Write non-MPI function not implemented."); } #if NGEN_WITH_MPI comm_ = MPI_COMM_NULL; @@ -80,13 +80,13 @@ int NetCDFManager::create_file(const std::string& filename) if (num_procs_ > 1) { // MPI-enabled NetCDF is_mpi_ = true; - nc_file_ = std::make_unique(filename, true, is_mpi_); + nc_file_ = std::make_unique(filename, NetCDFOpenMode::CREATE, is_mpi_); }else{ is_mpi_ = false; } #endif nc_filename_ = filename; - nc_file_ = std::make_unique(nc_filename_, true, is_mpi_); + nc_file_ = std::make_unique(nc_filename_, NetCDFOpenMode::CREATE, is_mpi_); if (!nc_file_) { throw std::runtime_error("Failed to create NetCDF file: " + filename); } diff --git a/test/core/NetCDFCreatorTest.cpp b/test/core/NetCDFCreatorTest.cpp index e7f383b563..f13b327fd2 100644 --- a/test/core/NetCDFCreatorTest.cpp +++ b/test/core/NetCDFCreatorTest.cpp @@ -213,7 +213,7 @@ TEST_F(NetCDFCreatorTest, TestCatchmentIdentifiers) { std::filesystem::path full_file_path = std::filesystem::temp_directory_path() / "catchment_test.nc"; std::string file_path = full_file_path.string(); - nc_manager = std::make_unique(manager_, file_path, *sim_time_, 0, 1); + nc_manager = std::make_unique(manager_, file_path, *sim_time_, NetCDFOpenMode::CREATE, 0, 1); NetCDFFile* nc_file = nc_manager->get_file_handle(); std::shared_ptr catchments_var = nc_file->get_ncvar("catchments"); size_t len = catchments_var->get_dim_size("catchments"); @@ -239,7 +239,7 @@ TEST_F(NetCDFCreatorTest, TestCatchmentOutputValues) { std::filesystem::path full_file_path = std::filesystem::temp_directory_path() / "catchment_test.nc"; std::string file_path = full_file_path.string(); - nc_manager = std::make_unique(manager_, file_path, *sim_time_, 0, 1); + nc_manager = std::make_unique(manager_, file_path, *sim_time_, NetCDFOpenMode::CREATE, 0, 1); NetCDFFile* nc_file = nc_manager->get_file_handle(); std::map catchment_output_values; auto c_form = std::dynamic_pointer_cast(manager_->get_formulation("cat-52")); From e23807a5255895700c76886d1c23cbeb6e981519 Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Fri, 17 Jul 2026 11:13:56 -0400 Subject: [PATCH 4/6] Simplify NetCDFManager simulation generation --- include/core/NgenSimulation.hpp | 2 +- include/netcdf/NetCDFManager.hpp | 2 +- src/NGen.cpp | 8 +++++--- src/core/NgenSimulation.cpp | 4 ++-- src/netcdf/NetCDFManager.cpp | 9 +++++---- test/core/NetCDFCreatorTest.cpp | 4 ++-- 6 files changed, 16 insertions(+), 13 deletions(-) diff --git a/include/core/NgenSimulation.hpp b/include/core/NgenSimulation.hpp index f011a52acf..b0366a779b 100644 --- a/include/core/NgenSimulation.hpp +++ b/include/core/NgenSimulation.hpp @@ -101,7 +101,7 @@ class NgenSimulation * @param merge_all_ranks Whether to also get the units from other MPI ranks. This should only be `true` when calling blocking MPI processes is safe for the program. */ std::vector required_checkpoint_units(bool merge_all_ranks) const; - void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, NetCDFOpenMode open_mode); + void create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, bool create_new_file); private: void advance_models_one_output_step(); diff --git a/include/netcdf/NetCDFManager.hpp b/include/netcdf/NetCDFManager.hpp index d8e3ad1bef..0df68de5e4 100644 --- a/include/netcdf/NetCDFManager.hpp +++ b/include/netcdf/NetCDFManager.hpp @@ -28,7 +28,7 @@ class NetCDFManager { public: NetCDFManager(std::shared_ptr manager, - const std::string& output_name, Simulation_Time const& sim_time, NetCDFOpenMode open_mode, int mpi_rank, int mpi_num_procs); + const std::string& output_name, Simulation_Time const& sim_time, bool create_new_file, int mpi_rank, int mpi_num_procs); // Constructor for read-only NetCDF (no MPI needed) NetCDFManager(const std::string& filename, NetCDFOpenMode open_mode); diff --git a/src/NGen.cpp b/src/NGen.cpp index 3c39598dab..d2721ba07e 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -728,7 +728,7 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { } #if NGEN_WITH_NETCDF - NetCDFOpenMode netcdf_open_mode = NetCDFOpenMode::CREATE; + bool create_new_netcdf_file = true; #endif // NGEN_WITH_NETCDF { // optionally load a checkpoint if configured auto checkpoint_loader = state_saving_config.checkpoint_loader(); @@ -739,13 +739,15 @@ int run_ngen(int argc, char* argv[], int mpi_num_procs, int mpi_rank) { = checkpoint_loader->initialize_checkpoint_snapshot(required_units); simulation->load_checkpoint(snapshot_loader); #if NGEN_WITH_NETCDF - netcdf_open_mode = NetCDFOpenMode::OPEN_WRITE; + create_new_netcdf_file = false; #endif //NGEN_WITH_NETCDF } } #if NGEN_WITH_NETCDF if (std::find(output_formats.begin(), output_formats.end(), "netcdf") != output_formats.end()) { - simulation->create_netcdf_writer(manager, "catchment_output", netcdf_open_mode); + if (!create_new_netcdf_file) + LOG(LogLevel::INFO, "Attempting to open the output of the prior checkpoint run's netCDF output file. The prior file must be located in the default output location to be correctly read."); + simulation->create_netcdf_writer(manager, "catchment_output", create_new_netcdf_file); } #endif //NGEN_WITH_NETCDF diff --git a/src/core/NgenSimulation.cpp b/src/core/NgenSimulation.cpp index c18a03cd06..25b6b11980 100644 --- a/src/core/NgenSimulation.cpp +++ b/src/core/NgenSimulation.cpp @@ -524,10 +524,10 @@ void NgenSimulation::serialize(Archive& ar, const unsigned int version) { #endif //NGEN_WITH_NEXUSES } -void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, NetCDFOpenMode open_mode) +void NgenSimulation::create_netcdf_writer(std::shared_ptr manager, std::string nc_output_file_name, bool create_new_file) { #if NGEN_WITH_NETCDF - this->nc_manager_ = std::make_unique(manager, nc_output_file_name, *sim_time_, open_mode, mpi_rank_, mpi_num_procs_); + this->nc_manager_ = std::make_unique(manager, nc_output_file_name, *sim_time_, create_new_file, mpi_rank_, mpi_num_procs_); #endif } diff --git a/src/netcdf/NetCDFManager.cpp b/src/netcdf/NetCDFManager.cpp index b0893bfe0e..92b2878a83 100644 --- a/src/netcdf/NetCDFManager.cpp +++ b/src/netcdf/NetCDFManager.cpp @@ -10,8 +10,8 @@ #include NetCDFManager::NetCDFManager(std::shared_ptr manager, - const std::string& output_name, Simulation_Time const& sim_time, NetCDFOpenMode open_mode, int mpi_rank, int mpi_num_procs) - : manager_{manager}, open_mode_{open_mode} + const std::string& output_name, Simulation_Time const& sim_time, bool create_new_file, int mpi_rank, int mpi_num_procs) + : manager_{manager}, open_mode_{create_new_file ? NetCDFOpenMode::CREATE : NetCDFOpenMode::OPEN_WRITE} { std::filesystem::path check_path(output_name); if(check_path.is_absolute()){ @@ -46,8 +46,9 @@ NetCDFManager::NetCDFManager(std::shared_ptr m } gather_all_catchments(catchments_in_proc); if (rank_ == 0){ - nc_file_ = std::make_unique(nc_filename_, open_mode, is_mpi_); - define_catchment_netcdf_components(sim_time); + nc_file_ = std::make_unique(nc_filename_, this->open_mode_, is_mpi_); + if (create_new_file) + define_catchment_netcdf_components(sim_time); } #if NGEN_WITH_MPI if (comm_ != MPI_COMM_NULL) //This check is important if the user runs MPI with a single process. diff --git a/test/core/NetCDFCreatorTest.cpp b/test/core/NetCDFCreatorTest.cpp index f13b327fd2..acdf2553d5 100644 --- a/test/core/NetCDFCreatorTest.cpp +++ b/test/core/NetCDFCreatorTest.cpp @@ -213,7 +213,7 @@ TEST_F(NetCDFCreatorTest, TestCatchmentIdentifiers) { std::filesystem::path full_file_path = std::filesystem::temp_directory_path() / "catchment_test.nc"; std::string file_path = full_file_path.string(); - nc_manager = std::make_unique(manager_, file_path, *sim_time_, NetCDFOpenMode::CREATE, 0, 1); + nc_manager = std::make_unique(manager_, file_path, *sim_time_, true, 0, 1); NetCDFFile* nc_file = nc_manager->get_file_handle(); std::shared_ptr catchments_var = nc_file->get_ncvar("catchments"); size_t len = catchments_var->get_dim_size("catchments"); @@ -239,7 +239,7 @@ TEST_F(NetCDFCreatorTest, TestCatchmentOutputValues) { std::filesystem::path full_file_path = std::filesystem::temp_directory_path() / "catchment_test.nc"; std::string file_path = full_file_path.string(); - nc_manager = std::make_unique(manager_, file_path, *sim_time_, NetCDFOpenMode::CREATE, 0, 1); + nc_manager = std::make_unique(manager_, file_path, *sim_time_, true, 0, 1); NetCDFFile* nc_file = nc_manager->get_file_handle(); std::map catchment_output_values; auto c_form = std::dynamic_pointer_cast(manager_->get_formulation("cat-52")); From 1ee6d73d90c612183bdf46bca148d415f88842ae Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 21 Jul 2026 12:16:39 -0400 Subject: [PATCH 5/6] Read variable definitions on checkpoint restart --- include/netcdf/NetCDFFile.hpp | 2 ++ include/netcdf/NetCDFManager.hpp | 2 ++ src/netcdf/NetCDFFile.cpp | 30 +++++++++++++++++++++ src/netcdf/NetCDFManager.cpp | 45 ++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+) diff --git a/include/netcdf/NetCDFFile.hpp b/include/netcdf/NetCDFFile.hpp index 2a3ad36814..2469b7d44b 100644 --- a/include/netcdf/NetCDFFile.hpp +++ b/include/netcdf/NetCDFFile.hpp @@ -35,6 +35,7 @@ class NetCDFFile { // Dimension handling int add_dimension(const std::string& name, size_t len); + int read_dimension_definition(const std::string& name); size_t get_dim_size(const std::string& name) const; int get_dim_id(const std::string& name) const; @@ -43,6 +44,7 @@ class NetCDFFile { std::vector list_variables() const; void add_ncvar(std::shared_ptr var); void add_variable(const std::string& name, nc_type type, const std::vector& dims, const std::vector& dim_names); + std::shared_ptr read_variable_definition(const std::string& name); // Write data to a variable template diff --git a/include/netcdf/NetCDFManager.hpp b/include/netcdf/NetCDFManager.hpp index 0df68de5e4..e7e1709e08 100644 --- a/include/netcdf/NetCDFManager.hpp +++ b/include/netcdf/NetCDFManager.hpp @@ -65,6 +65,7 @@ class NetCDFManager // Add variables to the file (for writing) void add_output_variable_data_from_formulation(); + void read_output_variable_data_from_formulation(); // Add catchment output data to the file (for writing) void write_simulations_response_from_formulation(size_t time_index, const std::map& catchment_output_values); @@ -77,6 +78,7 @@ class NetCDFManager /* Set up netcdf dimensions and variables. Note: A copy of Simulation_Time is passed because the object is expected to be modified and discarded at the end of the function. */ void define_catchment_netcdf_components(Simulation_Time sim_time); + void read_catchment_netcdf_components(); NetCDFOpenMode open_mode_; std::string nc_filename_; std::unique_ptr nc_file_; diff --git a/src/netcdf/NetCDFFile.cpp b/src/netcdf/NetCDFFile.cpp index 5e2252c2b9..93c5b927ec 100644 --- a/src/netcdf/NetCDFFile.cpp +++ b/src/netcdf/NetCDFFile.cpp @@ -57,6 +57,13 @@ int NetCDFFile::add_dimension(const std::string& name, size_t len) { return dimid; } +int NetCDFFile::read_dimension_definition(const std::string& name) { + int dimid; + NC_CHECK(nc_inq_dimid(ncid_, name.c_str(), &dimid), "NetCDF dimension " + name + " could not be found."); + dims_id_map_[name] = dimid; + return dimid; +} + void NetCDFFile::add_variable(const std::string& name, nc_type type, const std::vector& dim_ids, const std::vector& dim_names) { int varid; NC_CHECK(nc_def_var(ncid_, name.c_str(), type, dim_ids.size(), dim_ids.data(), &varid), "NetCDF variable definition failed"); @@ -70,6 +77,29 @@ void NetCDFFile::add_variable(const std::string& name, nc_type type, const std:: add_ncvar(var); } +std::shared_ptr NetCDFFile::read_variable_definition(const std::string& name) { + int varid, n_dim_ids; + nc_type type; + NC_CHECK(nc_inq_varid(ncid_, name.c_str(), &varid), "NetCDF variable " + name + " could not be found."); + NC_CHECK(nc_inq_vartype(ncid_, varid, &type), "NetCDF variable " + name + " type read failed."); + // get the variable's dimensions + std::vector dim_names; + std::vector dim_ids; + NC_CHECK(nc_inq_varndims(ncid_, varid, &n_dim_ids), "NetCDF variable " + name + " associated dimensions count read failed."); + dim_ids.resize(n_dim_ids); + dim_names.reserve(n_dim_ids); + NC_CHECK(nc_inq_vardimid(ncid_, varid, dim_ids.data()), "NetCDF variable " + name + " associated dimensions IDs read failed."); + char dim_name[NC_MAX_NAME + 1]; + for (const int dim_id : dim_ids) { + NC_CHECK(nc_inq_dimname(ncid_, dim_id, dim_name), "NetCDF dimension name query failed."); + std::string dim_str(dim_name); + dim_names.push_back(dim_str); + } + auto var = std::make_shared(name, type, dim_ids, dim_names, varid, ncid_); + add_ncvar(var); + return var; +} + // Get dimension length by name size_t NetCDFFile::get_dim_size(const std::string& name) const { auto it = dims_id_map_.find(name); diff --git a/src/netcdf/NetCDFManager.cpp b/src/netcdf/NetCDFManager.cpp index 92b2878a83..845c21919d 100644 --- a/src/netcdf/NetCDFManager.cpp +++ b/src/netcdf/NetCDFManager.cpp @@ -49,6 +49,8 @@ NetCDFManager::NetCDFManager(std::shared_ptr m nc_file_ = std::make_unique(nc_filename_, this->open_mode_, is_mpi_); if (create_new_file) define_catchment_netcdf_components(sim_time); + else + read_catchment_netcdf_components(); } #if NGEN_WITH_MPI if (comm_ != MPI_COMM_NULL) //This check is important if the user runs MPI with a single process. @@ -197,6 +199,26 @@ void NetCDFManager::define_catchment_netcdf_components(Simulation_Time sim_time) nc_file_->end_def_mode(); } +void NetCDFManager::read_catchment_netcdf_components() { + try { + this->nc_file_->read_dimension_definition("time"); + this->nc_file_->read_variable_definition("time"); + this->nc_file_->read_dimension_definition("catchments"); + this->nc_file_->read_variable_definition("catchments"); + } catch (const std::exception& e) { + LOG(LogLevel::FATAL, "The existing netCDF file must have existing dimensions and variables for 'time' and 'catchments'."); + LOG(LogLevel::FATAL, e.what()); + throw; + } + try { + this->read_output_variable_data_from_formulation(); + } catch (const std::exception& e) { + LOG(LogLevel::FATAL, "Failed to match netCDF output variables to forumation output variables."); + LOG(LogLevel::FATAL, e.what()); + throw; + } +} + int NetCDFManager::add_dimension(const std::string& name, size_t len){ return nc_file_->add_dimension(name, len); } @@ -229,6 +251,29 @@ void NetCDFManager::add_output_variable_data_from_formulation() } } +void NetCDFManager::read_output_variable_data_from_formulation() { + typename std::map>::const_iterator it = manager_->begin(); + const auto& catchment_info = *it; + auto r_c = std::dynamic_pointer_cast(catchment_info.second); + if(r_c->get_output_header_count() > 0){ + std::vectoroutput_variables = r_c->get_output_variable_names(); + std::vectoroutput_headers = r_c->get_output_header_fields(); + nc_output_variables_.reserve(output_variables.size()); + std::vector dim_names{"time", "catchments"}; + + for (int index = 0; index < output_variables.size(); index ++){ + auto var = nc_file_->read_variable_definition(output_headers[index]); + const auto& var_dim_names = var->get_dim_names(); + for (const std::string& var_dim : dim_names) + if (std::find(var_dim_names.begin(), var_dim_names.end(), var_dim) == var_dim_names.end()) + throw std::runtime_error("NetCDF output variable " + var->get_name() + " does not use dimension " + var_dim); + if (var->get_type() != NC_DOUBLE) + throw std::runtime_error("NetCDF output variable " + var->get_name() + " is not of type double."); + nc_output_variables_.push_back(output_headers[index]); + } + } +} + static std::vector string_split(std::string str, char delimiter) { std::stringstream ss(str); From 39576b9043a9217b5c18dbb873735a8b98335ced Mon Sep 17 00:00:00 2001 From: Ian Todd Date: Tue, 21 Jul 2026 13:58:44 -0400 Subject: [PATCH 6/6] Build catchment index on restart --- src/netcdf/NetCDFManager.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/netcdf/NetCDFManager.cpp b/src/netcdf/NetCDFManager.cpp index 845c21919d..cbe5259e39 100644 --- a/src/netcdf/NetCDFManager.cpp +++ b/src/netcdf/NetCDFManager.cpp @@ -200,16 +200,28 @@ void NetCDFManager::define_catchment_netcdf_components(Simulation_Time sim_time) } void NetCDFManager::read_catchment_netcdf_components() { + int cat_id; + std::shared_ptr cat_var; try { this->nc_file_->read_dimension_definition("time"); this->nc_file_->read_variable_definition("time"); - this->nc_file_->read_dimension_definition("catchments"); - this->nc_file_->read_variable_definition("catchments"); + cat_id = this->nc_file_->read_dimension_definition("catchments"); + cat_var = this->nc_file_->read_variable_definition("catchments"); } catch (const std::exception& e) { LOG(LogLevel::FATAL, "The existing netCDF file must have existing dimensions and variables for 'time' and 'catchments'."); LOG(LogLevel::FATAL, e.what()); throw; } + try { + size_t cat_size = cat_var->get_dim_size(cat_id); + if (cat_size != this->catchments_.size()) + throw std::runtime_error("The existing netCDF file has a different catchment dimension size than the run formulation."); + cat_var->build_catchments_index(cat_size); + } catch (const std::exception& e) { + LOG(LogLevel::FATAL, "NetCDF failed to build catchment output index."); + LOG(LogLevel::FATAL, e.what()); + throw; + } try { this->read_output_variable_data_from_formulation(); } catch (const std::exception& e) {