Fix lagging during metrics computation

blank_main_config.ini

@@ -1,6 +1,6 @@
 [Default]
-task=  # Task to perform, to sample from [study, validation]
-data_root=  # Path to the folder containing the raw dataset, organized according to the guidelines from the README.md
+task= # Task to perform, to sample from [study, validation]
+data_root= # Path to the folder containing the raw dataset, organized according to the guidelines from the README.md
 number_processes= # Number of processes to use in parallel for computation
 objective= # Validation task to run, to sample from [classification, segmentation]
 
@@ -14,20 +14,21 @@ selections_dense=  # List of strings separated with '\'. Each string should cont
 selections_categorical=  # Same as above, except that the second metric should be categorical. For example, MR sequence types with values [T1, T2, FLAIR].
 
 [Validation]
-input_folder=  # Path to the folder containing the prediction files from your model
-output_folder=  # Path to the folder where the validation results should be stored (will be created if non-existing)
+input_folder= # Path to the folder containing the prediction files from your model
+output_folder= # Path to the folder where the validation results should be stored (will be created if non-existing)
 gt_files_suffix= # Comma-separated list of strings for each class suffix, including file extension type (e.g., label_tumor.nii.gz)
-prediction_files_suffix=  # Comma-separated list of strings for each class suffix, including file extension type (e.g., pred_tumor.nii.gz)
-use_index_naming_convention=  # Boolean to indicate if the file naming convention with folder indexes is followed
-nb_folds=  # Integer value indicating the number of folds in the k-fold cross-validation
-split_way=  # String sampled from [two-way, three-way], to indicate if a train/val (two-way) or train/val/test (three-way) split is used for the k-fold cross-validation
-detection_overlap_thresholds=  # Comma-separated list of float, one value for each class, to indicate the minimum Dice overlap value for a segmentation to be considered valid
-metrics_space=  # Comma-separated list of spaces where to compute the metrics, to sample from: [pixelwise, patientwise, objectwise]
-extra_metrics=  # Comma-separated list of metrics to compute, to sample from [TPR, TNR, FPR, FNR, PPV, Jaccard, IOU, AUC, VS, VC, RAVD, GCE, MI, MCC, CKS, VOI, ARI, ASSD, HD95, MahaD, ProbD, OASSD]
-class_names=  # Comma-separated list of strings with the names of each segmented class
+prediction_files_suffix= # Comma-separated list of strings for each class suffix, including file extension type (e.g., pred_tumor.nii.gz)
+use_index_naming_convention= # Boolean to indicate if the file naming convention with folder indexes is followed
+nb_folds= # Integer value indicating the number of folds in the k-fold cross-validation
+split_way= # String sampled from [two-way, three-way], to indicate if a train/val (two-way) or train/val/test (three-way) split is used for the k-fold cross-validation
+detection_overlap_thresholds= # Comma-separated list of float, one value for each class, to indicate the minimum Dice overlap value for a segmentation to be considered valid
+metrics_space= # Comma-separated list of spaces where to compute the metrics, to sample from: [pixelwise, patientwise, objectwise]
+extra_metrics= # Comma-separated list of metrics to compute, to sample from [TPR, TNR, FPR, FNR, PPV, Jaccard, IOU, AUC, VS, VC, RAVD, GCE, MI, MCC, CKS, VOI, ARI, ASSD, HD95, MahaD, ProbD, OASSD]
+class_names= # Comma-separated list of strings with the names of each segmented class
 tiny_objects_removal_threshold= # Integer representing the minimum number of voxels an object must have to be kept as an object
 true_positive_volume_thresholds=  # Comma-separated list of float for cut-off values to apply to each class to consider them as true positives or not
 use_brats_data=
+results_save_frequency= # How often to flush results to CSV, sampled from [patient, fold]. Use 'fold' for large cohorts.
 
 [Standalone]
 groundtruth_filename=

raidionicsval/Utils/PatientMetricsStructure.py

@@ -102,20 +102,34 @@ def prediction_filepaths(self, prediction_filepaths) -> None:
 
     def init_from_file(self, study_folder: str):
         if self.objective == "segmentation":
-            self.__init_from_file_segmentation(study_folder=study_folder)
+            self.__init_from_file_or_df_segmentation(study_folder=study_folder)
         else:
             self.__init_from_file_classification(study_folder=study_folder)
 
-    def __init_from_file_segmentation(self, study_folder: str):
-        all_scores_filename = os.path.join(study_folder, 'all_dice_scores.csv')
+    def init_from_dataframes(self, all_scores_df: pd.DataFrame, class_scores_dfs: dict):
+        if self.objective == "segmentation":
+            self.__init_from_file_or_df_segmentation(all_scores_df=all_scores_df, class_scores_dfs=class_scores_dfs)
 
+    def __init_from_file_or_df_segmentation(self, study_folder: str = None, all_scores_df: pd.DataFrame = None,
+                                            class_scores_dfs: dict = None):
+
         for c in list(self._class_metrics.keys()):
-            class_scores_filename = os.path.join(study_folder, c + '_dice_scores.csv')
-            self._class_metrics[c].init_from_file(class_scores_filename)
-
-        if not os.path.exists(all_scores_filename):
+            if class_scores_dfs is not None:
+                self._class_metrics[c].init_from_file_or_df(df = class_scores_dfs[c])
+            else:
+                class_scores_filename = os.path.join(study_folder, c + '_dice_scores.csv')
+                self._class_metrics[c].init_from_file_or_df(scores_filename=class_scores_filename)
+
+        if all_scores_df is not None:
+            scores_df = all_scores_df
+        elif study_folder is not None:
+            all_scores_filename = os.path.join(study_folder, 'all_dice_scores.csv')
+            if not os.path.exists(all_scores_filename):
+                return
+            scores_df = pd.read_csv(all_scores_filename)
+        else:
             return
-        scores_df = pd.read_csv(all_scores_filename)
+
         scores_df['Patient'] = scores_df.Patient.astype(str)
         if len(scores_df.loc[(scores_df["Patient"] == self._patient_id) & (scores_df["Fold"] == self._fold_number)]) == 0:
             return
@@ -311,6 +325,7 @@ def set_results(self, results):
         self._patientwise_metrics = []
         self._pixelwise_metrics = []
         self._objectwise_metrics = []
+        self._extra_metrics = None
         for index in range(len(results)):
             thr_results = results[index][0]
             thr_val = thr_results[2]
@@ -322,16 +337,23 @@ def set_results(self, results):
             self._patientwise_metrics.append([thr_val] + patientwise_values)
             self._objectwise_metrics.append([thr_val] + objectwise_values)
 
-    def init_from_file(self, scores_filename: str) -> None:
-        if not os.path.exists(scores_filename):
+    def init_from_file_or_df(self, scores_filename: str = None, df: pd.DataFrame = None) -> None:
+        if df is not None:
+            scores_df = df
+        elif scores_filename is not None:
+            if not os.path.exists(scores_filename):
+                return
+            scores_df = pd.read_csv(scores_filename)
+        else:
             return
 
-        scores_df = pd.read_csv(scores_filename)
         scores_df['Patient'] = scores_df.Patient.astype(str)
-        if len(scores_df.loc[(scores_df["Patient"] == self._patient_id) & (scores_df["Fold"] == self._fold_number)]) == 0:
+        patient_class_scores = scores_df.loc[
+            (scores_df["Patient"] == self._patient_id) & (scores_df["Fold"] == self._fold_number)]
+
+        if len(patient_class_scores) == 0:
             return
 
-        patient_class_scores = scores_df.loc[(scores_df["Patient"] == self._patient_id) & (scores_df["Fold"] == self._fold_number)]
         self._patientwise_metrics = []
         self._pixelwise_metrics = []
         self._objectwise_metrics = []

raidionicsval/Utils/resources.py

@@ -64,6 +64,7 @@ def __setup(self):
         self.validation_class_names = []
         self.validation_true_positive_volume_thresholds = []
         self.validation_use_brats_data = []
+        self.validation_results_save_frequency = 'patient'
 
         self.standalone_gt_filename = None
         self.standalone_detection_filename = None
@@ -238,6 +239,13 @@ def __parse_validation_parameters(self):
             if self.config['Validation']['use_brats_data'].split('#')[0].strip() != '':
                 self.validation_use_brats_data = True if self.config['Validation']['use_brats_data'].split('#')[0].strip().lower() == 'true' else False
 
+        if self.config.has_option('Validation', 'results_save_frequency'):
+            if self.config['Validation']['results_save_frequency'].split('#')[0].strip() != '':
+                val = self.config['Validation']['results_save_frequency'].split('#')[0].strip()
+                if val not in ['patient', 'fold']:
+                    raise ValueError("results_save_frequency must be 'patient' or 'fold'")
+                self.validation_results_save_frequency = val
+
     def __parse_standalone_parameters(self):
         """