diff --git a/autofit/aggregator/summary/aggregate_fits.py b/autofit/aggregator/summary/aggregate_fits.py index 1f8267a98..56412aa4d 100644 --- a/autofit/aggregator/summary/aggregate_fits.py +++ b/autofit/aggregator/summary/aggregate_fits.py @@ -44,6 +44,12 @@ def _hdus( """ Extract the HDUs from a given fits for a given search. + Each source fits file is opened once (not once per requested HDU) and closed + deterministically — the extracted data is copied out of the memmap, so no + file handle stays open per result. Previously each HDU leaked one handle, + which exhausted the default open-file limit when aggregating many hundreds + of results. + Parameters ---------- result @@ -58,19 +64,27 @@ def _hdus( from astropy.io import fits row = [] - for hdu in hdus: - source = result.value(subplot_filename(hdu)) - source_hdu = source[source.index_of(hdu.value)] - - if extname_prefix is not None: - source_hdu.header["EXTNAME"] = f"{extname_prefix.upper()}_{source_hdu.header['EXTNAME']}" - - row.append( - fits.ImageHDU( - data=source_hdu.data, - header=source_hdu.header, + sources = {} + try: + for hdu in hdus: + source_name = subplot_filename(hdu) + if source_name not in sources: + sources[source_name] = result.value(source_name) + source = sources[source_name] + source_hdu = source[source.index_of(hdu.value)] + + if extname_prefix is not None: + source_hdu.header["EXTNAME"] = f"{extname_prefix.upper()}_{source_hdu.header['EXTNAME']}" + + row.append( + fits.ImageHDU( + data=source_hdu.data.copy(), + header=source_hdu.header, + ) ) - ) + finally: + for source in sources.values(): + source.close() return row def extract_fits(self, hdus: List[Enum], extname_prefix_list = None) -> "fits.HDUList": @@ -119,7 +133,11 @@ def extract_csv(self, filename: str) -> List[Dict]: output = [] for result in self.aggregator: - output.append(Table.read(result.value(filename), format="fits")) + source = result.value(filename) + try: + output.append(Table.read(source, format="fits").copy()) + finally: + source.close() return output diff --git a/test_autofit/aggregator/summary_files/test_aggregate_fits.py b/test_autofit/aggregator/summary_files/test_aggregate_fits.py index ba4aeb303..aa11d584c 100644 --- a/test_autofit/aggregator/summary_files/test_aggregate_fits.py +++ b/test_autofit/aggregator/summary_files/test_aggregate_fits.py @@ -57,3 +57,21 @@ def test_list_of_names(summary, output_directory): "one.fits", "two.fits", ]) + + +def test_extract_fits_closes_files(summary): + """ + Each source fits file is opened once per result and closed deterministically — + previously one handle leaked per requested HDU, exhausting the open-file limit + when aggregating many hundreds of results. + """ + import os + + if not os.path.isdir("/proc/self/fd"): + pytest.skip("file-descriptor counting requires /proc") + + before = len(os.listdir("/proc/self/fd")) + summary.extract_fits([FITSFit.ModelData, FITSFit.ResidualMap]) + after = len(os.listdir("/proc/self/fd")) + + assert after <= before + 1