From e36f9ece1c849906fb90e88300d5923ec30c539a Mon Sep 17 00:00:00 2001 From: Ragnhild Holden Date: Wed, 22 Apr 2026 14:18:23 +0200 Subject: [PATCH 1/3] Add option for saving once per fold, reduce df concats and lookups --- blank_main_config.ini | 25 +++-- raidionicsval/Utils/resources.py | 8 ++ .../Validation/kfold_model_validation.py | 103 +++++++++++++----- 3 files changed, 96 insertions(+), 40 deletions(-) diff --git a/blank_main_config.ini b/blank_main_config.ini index c56a764..99e4ab0 100644 --- a/blank_main_config.ini +++ b/blank_main_config.ini @@ -1,6 +1,6 @@ [Default] -task= # Task to perform, to sample from [study, validation] -data_root= # Path to the folder containing the raw dataset, organized according to the guidelines from the README.md +task= # Task to perform, to sample from [study, validation] +data_root= # Path to the folder containing the raw dataset, organized according to the guidelines from the README.md number_processes= # Number of processes to use in parallel for computation objective= # Validation task to run, to sample from [classification, segmentation] @@ -14,20 +14,21 @@ selections_dense= # List of strings separated with '\'. Each string should cont selections_categorical= # Same as above, except that the second metric should be categorical. For example, MR sequence types with values [T1, T2, FLAIR]. [Validation] -input_folder= # Path to the folder containing the prediction files from your model -output_folder= # Path to the folder where the validation results should be stored (will be created if non-existing) +input_folder= # Path to the folder containing the prediction files from your model +output_folder= # Path to the folder where the validation results should be stored (will be created if non-existing) gt_files_suffix= # Comma-separated list of strings for each class suffix, including file extension type (e.g., label_tumor.nii.gz) -prediction_files_suffix= # Comma-separated list of strings for each class suffix, including file extension type (e.g., pred_tumor.nii.gz) -use_index_naming_convention= # Boolean to indicate if the file naming convention with folder indexes is followed -nb_folds= # Integer value indicating the number of folds in the k-fold cross-validation -split_way= # String sampled from [two-way, three-way], to indicate if a train/val (two-way) or train/val/test (three-way) split is used for the k-fold cross-validation -detection_overlap_thresholds= # Comma-separated list of float, one value for each class, to indicate the minimum Dice overlap value for a segmentation to be considered valid -metrics_space= # Comma-separated list of spaces where to compute the metrics, to sample from: [pixelwise, patientwise, objectwise] -extra_metrics= # Comma-separated list of metrics to compute, to sample from [TPR, TNR, FPR, FNR, PPV, Jaccard, IOU, AUC, VS, VC, RAVD, GCE, MI, MCC, CKS, VOI, ARI, ASSD, HD95, MahaD, ProbD, OASSD] -class_names= # Comma-separated list of strings with the names of each segmented class +prediction_files_suffix= # Comma-separated list of strings for each class suffix, including file extension type (e.g., pred_tumor.nii.gz) +use_index_naming_convention= # Boolean to indicate if the file naming convention with folder indexes is followed +nb_folds= # Integer value indicating the number of folds in the k-fold cross-validation +split_way= # String sampled from [two-way, three-way], to indicate if a train/val (two-way) or train/val/test (three-way) split is used for the k-fold cross-validation +detection_overlap_thresholds= # Comma-separated list of float, one value for each class, to indicate the minimum Dice overlap value for a segmentation to be considered valid +metrics_space= # Comma-separated list of spaces where to compute the metrics, to sample from: [pixelwise, patientwise, objectwise] +extra_metrics= # Comma-separated list of metrics to compute, to sample from [TPR, TNR, FPR, FNR, PPV, Jaccard, IOU, AUC, VS, VC, RAVD, GCE, MI, MCC, CKS, VOI, ARI, ASSD, HD95, MahaD, ProbD, OASSD] +class_names= # Comma-separated list of strings with the names of each segmented class tiny_objects_removal_threshold= # Integer representing the minimum number of voxels an object must have to be kept as an object true_positive_volume_thresholds= # Comma-separated list of float for cut-off values to apply to each class to consider them as true positives or not use_brats_data= +results_save_frequency= # How often to flush results to CSV, sampled from [patient, fold]. Use 'fold' for large cohorts. [Standalone] groundtruth_filename= diff --git a/raidionicsval/Utils/resources.py b/raidionicsval/Utils/resources.py index 5662532..771c2df 100644 --- a/raidionicsval/Utils/resources.py +++ b/raidionicsval/Utils/resources.py @@ -64,6 +64,7 @@ def __setup(self): self.validation_class_names = [] self.validation_true_positive_volume_thresholds = [] self.validation_use_brats_data = [] + self.validation_results_save_frequency = 'patient' self.standalone_gt_filename = None self.standalone_detection_filename = None @@ -238,6 +239,13 @@ def __parse_validation_parameters(self): if self.config['Validation']['use_brats_data'].split('#')[0].strip() != '': self.validation_use_brats_data = True if self.config['Validation']['use_brats_data'].split('#')[0].strip().lower() == 'true' else False + if self.config.has_option('Validation', 'results_save_frequency'): + if self.config['Validation']['results_save_frequency'].split('#')[0].strip() != '': + val = self.config['Validation']['results_save_frequency'].split('#')[0].strip() + if val not in ['patient', 'fold']: + raise ValueError("results_save_frequency must be 'patient' or 'fold'") + self.validation_results_save_frequency = val + def __parse_standalone_parameters(self): """ diff --git a/raidionicsval/Validation/kfold_model_validation.py b/raidionicsval/Validation/kfold_model_validation.py index 00c75c0..d9ce789 100644 --- a/raidionicsval/Validation/kfold_model_validation.py +++ b/raidionicsval/Validation/kfold_model_validation.py @@ -119,6 +119,19 @@ def __compute_metrics(self): for c in SharedResources.getInstance().validation_class_names: self.class_results_df[c]['Patient'] = self.class_results_df[c].Patient.astype(str) + # Create idx lookup table to save .loc scans in the resume check + self._key_to_idx = {} + for c in SharedResources.getInstance().validation_class_names: + self._key_to_idx[c] = { + (row['Patient'], row['Fold'], row['Threshold']): i for i, row in self.class_results_df[c].iterrows()} + + self._key_to_idx_overall ={ + (row['Patient'], row['Fold'], row['Threshold']): i for i, row in self.results_df.iterrows()} + + # Pending row accumulators — avoids per-patient pd.concat + self._pending_class_rows = {c: [] for c in SharedResources.getInstance().validation_class_names} + self._pending_overall_rows = [] + results_per_folds = [] for fold in range(0, self.fold_number): logging.info(f'\nProcessing fold {fold+1}/{self.fold_number}.\n') @@ -168,6 +181,18 @@ def __compute_metrics_for_fold(self, data_list, fold_number): print('Issue processing patient {}\n'.format(uid)) print(traceback.format_exc()) continue + + if SharedResources.getInstance().validation_results_save_frequency == 'fold': + if self._pending_overall_rows: + self.results_df = pd.concat([self.results_df] + self._pending_overall_rows, ignore_index=True) + self._pending_overall_rows = [] + self.results_df.to_csv(self.dice_output_filename, index=False) + for c in SharedResources.getInstance().validation_class_names: + if self._pending_class_rows[c]: + self.class_results_df[c] = pd.concat( + [self.class_results_df[c]] + self._pending_class_rows[c], ignore_index = True) + self._pending_class_rows[c] = [] + self.class_results_df[c].to_csv(self.class_dice_output_filenames[c], index=False) return 0 def __identify_patient_files(self, patient_metrics: PatientMetrics, folder_index: int, fold_number: int) -> bool: @@ -359,27 +384,35 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): patient_metrics.set_class_regular_metrics(classes[c], pat_results) # Filling in the csv files on disk for faster resume class_results_filename = self.class_dice_output_filenames[classes[c]] + new_class_rows = [] for ind, th in enumerate(thr_range): th = np.round(th, 2) - sub_df = self.class_results_df[classes[c]].loc[ - (self.class_results_df[classes[c]]['Patient'] == uid) & (self.class_results_df[classes[c]]['Fold'] == fold_number) & ( - self.class_results_df[classes[c]]['Threshold'] == th)] - # ind_values = np.asarray(pat_results[ind]) - # buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), - # columns=list(self.results_df_base_columns)) - if len(sub_df) == 0: + # sub_df = self.class_results_df[classes[c]].loc[ + # (self.class_results_df[classes[c]]['Patient'] == uid) & (self.class_results_df[classes[c]]['Fold'] == fold_number) & ( + # self.class_results_df[classes[c]]['Threshold'] == th)] + key = (uid, fold_number, th) + if key not in self._key_to_idx[classes[c]]: extra_metrics = [None] * 2 * len(SharedResources.getInstance().validation_metric_names) ind_values = np.asarray(pat_results[ind][0] + extra_metrics) buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), columns=list(self.results_df_base_columns)) - # self.class_results_df[classes[c]] = self.class_results_df[classes[c]].append(buff_df, - # ignore_index=True) - self.class_results_df[classes[c]] = pd.concat([self.class_results_df[classes[c]], buff_df], - ignore_index=True) + new_class_rows.append(buff_df) else: - ind_values = pat_results[ind][0] + list(self.class_results_df[classes[c]].loc[sub_df.index.values[0], :].values[len(pat_results[ind][0]):]) - self.class_results_df[classes[c]].loc[sub_df.index.values[0], :] = ind_values - self.class_results_df[classes[c]].to_csv(class_results_filename, index=False) + row_idx = self._key_to_idx[classes[c]][key] + ind_values = pat_results[ind][0] + list( + self.class_results_df[classes[c]].loc[row_idx, :].values[len(pat_results[ind][0]):]) + self.class_results_df[classes[c]].loc[row_idx, :] = ind_values + self._pending_class_rows[classes[c]].extend(new_class_rows) + # if new_class_rows: + # self.class_results_df[classes[c]] = pd.concat([self.class_results_df[classes[c]]] + new_class_rows, + # ignore_index=True) + + # self.class_results_df[classes[c]].to_csv(class_results_filename, index=False) + if SharedResources.getInstance().validation_results_save_frequency == 'patient': + self.class_results_df[classes[c]] = pd.concat( + [self.class_results_df[classes[c]]] + self._pending_class_rows[classes[c]], ignore_index=True) + self._pending_class_rows[classes[c]] = [] + self.class_results_df[classes[c]].to_csv(class_results_filename, index=False) # Should compute the class macro-average results if multiple classes class_averaged_results = None @@ -394,24 +427,28 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): class_averaged_results = np.average(np.asarray(class_results).astype('float32')[:, :, 1:], axis=0) # Filling in the csv files on disk for faster resume + new_overall_rows = [] for ind, th in enumerate(thr_range): th = np.round(th, 2) - sub_df = self.results_df.loc[ - (self.results_df['Patient'] == uid) & (self.results_df['Fold'] == fold_number) & ( - self.results_df['Threshold'] == th)] - # ind_values = np.asarray([fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind])) - # buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), - # columns=list(self.results_df_base_columns)) - if len(sub_df) == 0: + key = (uid, fold_number, th) + # sub_df = self.results_df.loc[ + # (self.results_df['Patient'] == uid) & (self.results_df['Fold'] == fold_number) & ( + # self.results_df['Threshold'] == th)] + if key not in self._key_to_idx_overall: ind_values = np.asarray([fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind])) buff_df = pd.DataFrame(ind_values.reshape(1, len(self.results_df_base_columns)), columns=list(self.results_df_base_columns)) - # self.results_df = self.results_df.append(buff_df, ignore_index=True) - self.results_df = pd.concat([self.results_df, buff_df], ignore_index=True) + new_overall_rows.append(buff_df) else: + row_idx = self._key_to_idx_overall[key] ind_values = [fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind]) - self.results_df.loc[sub_df.index.values[0], :] = ind_values - self.results_df.to_csv(self.dice_output_filename, index=False) + self.results_df.loc[row_idx :] = ind_values + + self._pending_overall_rows.extend(new_overall_rows) + if SharedResources.getInstance().validation_results_save_frequency == 'patient': + self.results_df = pd.concat([self.results_df] + self._pending_overall_rows, ignore_index=True) + self._pending_overall_rows = [] + self.results_df.to_csv(self.dice_output_filename, index=False) def __compute_extra_metrics(self, class_optimal: dict = {}): """ @@ -420,6 +457,10 @@ def __compute_extra_metrics(self, class_optimal: dict = {}): classes = SharedResources.getInstance().validation_class_names for c in classes: optimal_values = class_optimal[c]['All'] + idx_lookup = { + (row['Patient'], row['Threshold']): i + for i, row in self.class_results_df[c].iterrows() + } for p in tqdm(self.patients_metrics): try: # Initializing/completing the list which will hold the extra metrics @@ -432,8 +473,14 @@ def __compute_extra_metrics(self, class_optimal: dict = {}): for pm in pat_metrics: metric_name = pm[0] metric_value = pm[1] - self.class_results_df[c].at[self.class_results_df[c].loc[(self.class_results_df[c]['Patient'] == self.patients_metrics[p].patient_id) & (self.class_results_df[c]['Threshold'] == optimal_values[1])].index.values[0], metric_name] = metric_value - self.class_results_df[c].to_csv(self.class_dice_output_filenames[c], index=False) + row_idx = idx_lookup[(self.patients_metrics[p].patient_id, optimal_values[1])] + self.class_results_df[c].at[row_idx, metric_name] = metric_value + # self.class_results_df[c].at[self.class_results_df[c].loc[(self.class_results_df[c]['Patient'] == self.patients_metrics[p].patient_id) & (self.class_results_df[c]['Threshold'] == optimal_values[1])].index.values[0], metric_name] = metric_value + if SharedResources.getInstance().validation_results_save_frequency == 'patient': + self.class_results_df[c].to_csv(self.class_dice_output_filenames[c], index=False) except Exception as e: logging.error(f"Computing extra metrics for patient {self.patients_metrics[p].patient_id} failed with: {e}.\n{traceback.format_exc()}") - continue \ No newline at end of file + continue + + if SharedResources.getInstance().validation_results_save_frequency == 'fold': + self.class_results_df[c].to_csv(self.class_dice_output_filenames[c], index=False) \ No newline at end of file From 658ae7a0d66e5fabb87c0e94c29c009f30f59580 Mon Sep 17 00:00:00 2001 From: Ragnhild Holden Date: Thu, 23 Apr 2026 11:09:05 +0200 Subject: [PATCH 2/3] Fix consistency problems --- .../Utils/PatientMetricsStructure.py | 48 ++++++++++++----- .../Validation/kfold_model_validation.py | 52 ++++++++++++++++--- 2 files changed, 81 insertions(+), 19 deletions(-) diff --git a/raidionicsval/Utils/PatientMetricsStructure.py b/raidionicsval/Utils/PatientMetricsStructure.py index 2f3fbe0..54a0a7a 100644 --- a/raidionicsval/Utils/PatientMetricsStructure.py +++ b/raidionicsval/Utils/PatientMetricsStructure.py @@ -102,20 +102,34 @@ def prediction_filepaths(self, prediction_filepaths) -> None: def init_from_file(self, study_folder: str): if self.objective == "segmentation": - self.__init_from_file_segmentation(study_folder=study_folder) + self.__init_from_file_or_df_segmentation(study_folder=study_folder) else: self.__init_from_file_classification(study_folder=study_folder) - def __init_from_file_segmentation(self, study_folder: str): - all_scores_filename = os.path.join(study_folder, 'all_dice_scores.csv') + def init_from_dataframes(self, all_scores_df: pd.DataFrame, class_scores_dfs: dict): + if self.objective == "segmentation": + self.__init_from_file_or_df_segmentation(all_scores_df=all_scores_df, class_scores_dfs=class_scores_dfs) + def __init_from_file_or_df_segmentation(self, study_folder: str = None, all_scores_df: pd.DataFrame = None, + class_scores_dfs: dict = None): + for c in list(self._class_metrics.keys()): - class_scores_filename = os.path.join(study_folder, c + '_dice_scores.csv') - self._class_metrics[c].init_from_file(class_scores_filename) - - if not os.path.exists(all_scores_filename): + if class_scores_dfs is not None: + self._class_metrics[c].init_from_file_or_df(df = class_scores_dfs[c]) + else: + class_scores_filename = os.path.join(study_folder, c + '_dice_scores.csv') + self._class_metrics[c].init_from_file_or_df(scores_filename=class_scores_filename) + + if all_scores_df is not None: + scores_df = all_scores_df + elif study_folder is not None: + all_scores_filename = os.path.join(study_folder, 'all_dice_scores.csv') + if not os.path.exists(all_scores_filename): + return + scores_df = pd.read_csv(all_scores_filename) + else: return - scores_df = pd.read_csv(all_scores_filename) + scores_df['Patient'] = scores_df.Patient.astype(str) if len(scores_df.loc[(scores_df["Patient"] == self._patient_id) & (scores_df["Fold"] == self._fold_number)]) == 0: return @@ -311,6 +325,7 @@ def set_results(self, results): self._patientwise_metrics = [] self._pixelwise_metrics = [] self._objectwise_metrics = [] + self._extra_metrics = None for index in range(len(results)): thr_results = results[index][0] thr_val = thr_results[2] @@ -322,16 +337,23 @@ def set_results(self, results): self._patientwise_metrics.append([thr_val] + patientwise_values) self._objectwise_metrics.append([thr_val] + objectwise_values) - def init_from_file(self, scores_filename: str) -> None: - if not os.path.exists(scores_filename): + def init_from_file_or_df(self, scores_filename: str = None, df: pd.DataFrame = None) -> None: + if df is not None: + scores_df = df + elif scores_filename is not None: + if not os.path.exists(scores_filename): + return + scores_df = pd.read_csv(scores_filename) + else: return - scores_df = pd.read_csv(scores_filename) scores_df['Patient'] = scores_df.Patient.astype(str) - if len(scores_df.loc[(scores_df["Patient"] == self._patient_id) & (scores_df["Fold"] == self._fold_number)]) == 0: + patient_class_scores = scores_df.loc[ + (scores_df["Patient"] == self._patient_id) & (scores_df["Fold"] == self._fold_number)] + + if len(patient_class_scores) == 0: return - patient_class_scores = scores_df.loc[(scores_df["Patient"] == self._patient_id) & (scores_df["Fold"] == self._fold_number)] self._patientwise_metrics = [] self._pixelwise_metrics = [] self._objectwise_metrics = [] diff --git a/raidionicsval/Validation/kfold_model_validation.py b/raidionicsval/Validation/kfold_model_validation.py index d9ce789..73d959b 100644 --- a/raidionicsval/Validation/kfold_model_validation.py +++ b/raidionicsval/Validation/kfold_model_validation.py @@ -98,8 +98,9 @@ def __compute_metrics(self): self.results_df = pd.read_csv(self.dice_output_filename) if self.results_df.columns[0] != 'Fold': self.results_df = pd.read_csv(self.dice_output_filename, index_col=0) - missing_metrics = [x for x in SharedResources.getInstance().validation_metric_names if - not x in list(self.results_df.columns)[1:]] + # missing_metrics = [x for x in SharedResources.getInstance().validation_metric_names if + # not x in list(self.results_df.columns)[1:]] + missing_metrics = [x for x in self.metric_names if not x in list(self.results_df.columns)[1:]] for m in missing_metrics: self.results_df[m] = None @@ -132,6 +133,40 @@ def __compute_metrics(self): self._pending_class_rows = {c: [] for c in SharedResources.getInstance().validation_class_names} self._pending_overall_rows = [] + # Remove patients that are inconsistent across CSVs (present in some but not all) + # so they are fully recomputed rather than partially updated in-place + classes = SharedResources.getInstance().validation_class_names + + def patient_fold_keys(key_dict): + return {(k[0], k[1]) for k in key_dict} + + overall_pf = patient_fold_keys(self._key_to_idx_overall) + class_pf = {c: patient_fold_keys(self._key_to_idx[c]) for c in classes} + all_pf = overall_pf.intersection(*class_pf.values()) + inconsistent_pf = overall_pf.union(*class_pf.values()) - all_pf + + if inconsistent_pf: + logging.warning( + f"{len(inconsistent_pf)} patient-fold combinations are inconsistent across CSVs and will be fully recomputed.") + + def drop_pf(df): + mask = pd.Series(list(zip(df['Patient'].astype(str), df['Fold']))).isin(inconsistent_pf).values + return df[~mask].reset_index(drop=True) + + self.results_df = drop_pf(self.results_df) + for c in classes: + self.class_results_df[c] = drop_pf(self.class_results_df[c]) + + self._key_to_idx_overall = { + (row['Patient'], row['Fold'], row['Threshold']): i + for i, row in self.results_df.iterrows() + } + for c in classes: + self._key_to_idx[c] = { + (row['Patient'], row['Fold'], row['Threshold']): i + for i, row in self.class_results_df[c].iterrows() + } + results_per_folds = [] for fold in range(0, self.fold_number): logging.info(f'\nProcessing fold {fold+1}/{self.fold_number}.\n') @@ -164,14 +199,16 @@ def __compute_metrics_for_fold(self, data_list, fold_number): # Placeholder for holding all metrics for the current patient patient_metrics = PatientMetrics(id=uid, patient_id=pid, fold_number=fold_number, class_names=SharedResources.getInstance().validation_class_names) - patient_metrics.init_from_file(self.output_folder) + # patient_metrics.init_from_file(self.output_folder) + patient_metrics.init_from_dataframes(self.results_df, self.class_results_df) + self.patients_metrics[uid] = patient_metrics success = self.__identify_patient_files(patient_metrics, sub_folder_index, fold_number) - self.patients_metrics[uid] = patient_metrics # Checking if values have already been computed for the current patient to skip it if so. if patient_metrics.is_complete(): continue + if not success: print('Input files not found for patient {}\n'.format(uid)) continue @@ -441,8 +478,11 @@ def __generate_dice_scores_for_patient(self, patient_metrics, fold_number): new_overall_rows.append(buff_df) else: row_idx = self._key_to_idx_overall[key] - ind_values = [fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind]) - self.results_df.loc[row_idx :] = ind_values + # ind_values = [fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind]) + # self.results_df.loc[row_idx, :] = ind_values + base_values = [fold_number, uid, np.round(th, 2)] + list(class_averaged_results[ind]) + existing_trailing = list(self.results_df.loc[row_idx, :].values[len(base_values):]) + self.results_df.loc[row_idx, :] = base_values + existing_trailing self._pending_overall_rows.extend(new_overall_rows) if SharedResources.getInstance().validation_results_save_frequency == 'patient': From a911faa5a87c91ff56a6fc84bc6b4e8a3574b553 Mon Sep 17 00:00:00 2001 From: Ragnhild Holden Date: Thu, 23 Apr 2026 17:13:08 +0200 Subject: [PATCH 3/3] Add explaining comments --- raidionicsval/Validation/kfold_model_validation.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/raidionicsval/Validation/kfold_model_validation.py b/raidionicsval/Validation/kfold_model_validation.py index 73d959b..3367501 100644 --- a/raidionicsval/Validation/kfold_model_validation.py +++ b/raidionicsval/Validation/kfold_model_validation.py @@ -137,6 +137,8 @@ def __compute_metrics(self): # so they are fully recomputed rather than partially updated in-place classes = SharedResources.getInstance().validation_class_names + # Check for inconsistensies that can occur if the computation is stopped, if metrics are saved for one patient + # and one class but not all for example def patient_fold_keys(key_dict): return {(k[0], k[1]) for k in key_dict} @@ -199,8 +201,11 @@ def __compute_metrics_for_fold(self, data_list, fold_number): # Placeholder for holding all metrics for the current patient patient_metrics = PatientMetrics(id=uid, patient_id=pid, fold_number=fold_number, class_names=SharedResources.getInstance().validation_class_names) + + # init_from_dataframes only works for segmentation so far # patient_metrics.init_from_file(self.output_folder) patient_metrics.init_from_dataframes(self.results_df, self.class_results_df) + self.patients_metrics[uid] = patient_metrics success = self.__identify_patient_files(patient_metrics, sub_folder_index, fold_number)