fix: AggregateFITS leaked one file handle per HDU per result (crash at ~500 results)

autofit/aggregator/summary/aggregate_fits.py

@@ -44,6 +44,12 @@ def _hdus(
         """
         Extract the HDUs from a given fits for a given search.
 
+        Each source fits file is opened once (not once per requested HDU) and closed
+        deterministically — the extracted data is copied out of the memmap, so no
+        file handle stays open per result. Previously each HDU leaked one handle,
+        which exhausted the default open-file limit when aggregating many hundreds
+        of results.
+
         Parameters
         ----------
         result
@@ -58,19 +64,27 @@ def _hdus(
         from astropy.io import fits
 
         row = []
-        for hdu in hdus:
-            source = result.value(subplot_filename(hdu))
-            source_hdu = source[source.index_of(hdu.value)]
-
-            if extname_prefix is not None:
-                source_hdu.header["EXTNAME"] = f"{extname_prefix.upper()}_{source_hdu.header['EXTNAME']}"
-
-            row.append(
-                fits.ImageHDU(
-                    data=source_hdu.data,
-                    header=source_hdu.header,
+        sources = {}
+        try:
+            for hdu in hdus:
+                source_name = subplot_filename(hdu)
+                if source_name not in sources:
+                    sources[source_name] = result.value(source_name)
+                source = sources[source_name]
+                source_hdu = source[source.index_of(hdu.value)]
+
+                if extname_prefix is not None:
+                    source_hdu.header["EXTNAME"] = f"{extname_prefix.upper()}_{source_hdu.header['EXTNAME']}"
+
+                row.append(
+                    fits.ImageHDU(
+                        data=source_hdu.data.copy(),
+                        header=source_hdu.header,
+                    )
                 )
-            )
+        finally:
+            for source in sources.values():
+                source.close()
         return row
 
     def extract_fits(self, hdus: List[Enum], extname_prefix_list = None) -> "fits.HDUList":
@@ -119,7 +133,11 @@ def extract_csv(self, filename: str) -> List[Dict]:
 
         output = []
         for result in self.aggregator:
-            output.append(Table.read(result.value(filename), format="fits"))
+            source = result.value(filename)
+            try:
+                output.append(Table.read(source, format="fits").copy())
+            finally:
+                source.close()
 
         return output
 

test_autofit/aggregator/summary_files/test_aggregate_fits.py

@@ -57,3 +57,21 @@ def test_list_of_names(summary, output_directory):
         "one.fits",
         "two.fits",
     ])
+
+
+def test_extract_fits_closes_files(summary):
+    """
+    Each source fits file is opened once per result and closed deterministically —
+    previously one handle leaked per requested HDU, exhausting the open-file limit
+    when aggregating many hundreds of results.
+    """
+    import os
+
+    if not os.path.isdir("/proc/self/fd"):
+        pytest.skip("file-descriptor counting requires /proc")
+
+    before = len(os.listdir("/proc/self/fd"))
+    summary.extract_fits([FITSFit.ModelData, FITSFit.ResidualMap])
+    after = len(os.listdir("/proc/self/fd"))
+
+    assert after <= before + 1