diff --git a/AGENTS.md b/AGENTS.md index 11e3a75b..2d544550 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,9 +26,10 @@ ecosystem via the CTI resurrection epic visualization layer rewritten on the matplotlib **function API**, mirroring PyAutoGalaxy: per-domain `plot/*_plots.py` function modules, config-gated `model/plotter.py` orchestrators, `autocti/util/plot_utils.py` helpers) are -complete. Remaining: Phase 2 autofit sync (5 aggregator tests skipped pending -the `AnalysisFactor`/`FactorGraphModel` port), Phase 3 CI + ecosystem -plumbing, Phase 4 workspace update, Phase 5 workspace_test rebuild + release. +complete, as is Phase 2 (autofit sync: multi-dataset fits and the aggregator +run through `af.AnalysisFactor`/`af.FactorGraphModel`; the test suite has no +skips). Remaining: Phase 3 CI + ecosystem plumbing, Phase 4 workspace update, +Phase 5 workspace_test rebuild + release. ## arcticpy (read before installing) diff --git a/autocti/aggregator/fit_dataset_1d.py b/autocti/aggregator/fit_dataset_1d.py index 1d8ddc35..d6b56149 100644 --- a/autocti/aggregator/fit_dataset_1d.py +++ b/autocti/aggregator/fit_dataset_1d.py @@ -1,159 +1,183 @@ -from __future__ import annotations -from typing import TYPE_CHECKING, List, Optional - -from autocti.aggregator.abstract import AggBase - -if TYPE_CHECKING: - from autocti.clocker.abstract import AbstractClocker - from autocti.dataset_1d.fit import FitDataset1D - -import autofit as af - -from autocti.aggregator.dataset_1d import _dataset_1d_list_from - - -def _fit_dataset_1d_list_from( - fit: af.Fit, - instance: Optional[af.ModelInstance] = None, - use_dataset_full: bool = False, - clocker_list: Optional[AbstractClocker] = None, -) -> List[FitDataset1D]: - """ - Returns a list of `FitDataset1D` object from a `PyAutoFit` sqlite database `Fit` object. - - The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit: - - - The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder). - - The clocker used to add CTI in the fit (`dataset/clocker.json`). - - The settings used for clocking CIT (contained in `dataset/settings_cti.json`). - - Each individual attribute can be loaded from the database via the `fit.value()` method. - - This method combines all of these attributes and returns a list of `FitDataset1D` objects, by loading the masked - dataset adding CTI to its pre-cti data via the cti model and clocking and fitting the model image to the dataset. - - If multiple `Dataset1D` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method - is instead used to load lists of the datasets, perform the fit and return a list of `FitDataset1D` objects. - - If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible - to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. - - Parameters - ---------- - fit - A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database. - instance - A manual instance that overwrites the max log likelihood instance in fit (e.g. for drawing the instance - randomly from the PDF). - use_dataset_full - If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible - to the database, the input `use_dataset_full` can be switched in to load instead the full `Dataset1D` objects. - clocker_list - If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit. - """ - - from autocti.dataset_1d.fit import FitDataset1D - - dataset_list = _dataset_1d_list_from(fit=fit, use_dataset_full=use_dataset_full) - - if clocker_list is None: - if not fit.children: - clocker_list = [fit.value(name="clocker")] - else: - clocker_list = fit.child_values(name="clocker") - - if instance is not None: - cti = instance.cti - else: - cti = fit.instance.cti - - post_cti_data_list = [ - clocker.add_cti(data=dataset.pre_cti_data, cti=cti) - for dataset, clocker in zip(dataset_list, clocker_list) - ] - - return [ - FitDataset1D( - dataset=dataset, - post_cti_data=post_cti_data, - ) - for dataset, post_cti_data in zip(dataset_list, post_cti_data_list) - ] - - -class FitDataset1DAgg(AggBase): - def __init__( - self, - aggregator: af.Aggregator, - use_dataset_full: bool = False, - clocker_list: Optional[List[AbstractClocker]] = None, - ): - """ - Interfaces with an `PyAutoFit` aggregator object to create instances of `Dataset1D` objects from the results - of a model-fit. - - The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit: - - - The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder). - - The clocker used to add CTI in the fit (`dataset/clocker.json`). - - The settings used for clocking CIT (contained in `dataset/settings_cti.json`). - - The `aggregator` contains the path to each of these files, and they can be loaded individually. This class - can load them all at once and create a `FitDataset1D` object via the `_fit_dataset_1d_from` method. - - This class's methods returns generators which create the instances of the `FitDataset1D` objects. This ensures - that large sets of results can be efficiently loaded from the hard-disk and do not require storing all - `Dataset1D` instances in the memory at once. - - For example, if the `aggregator` contains 3 model-fits, this class can be used to create a generator which - creates instances of the corresponding 3 `Dataset1D` objects. - - If multiple `Dataset1D` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method - is instead used to load lists of the datasets, perform the fit and return a list of `FitDataset1D` objects. - - If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible - to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. - - This can be done manually, but this object provides a more concise API. - - Parameters - ---------- - aggregator - A `PyAutoFit` aggregator object which can load the results of model-fits. - use_dataset_full - If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore - accessible to the database, the input `use_dataset_full` can be switched in to load instead the - full `Dataset1D` objects. - clocker_list - If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit. - """ - super().__init__( - aggregator=aggregator, - use_dataset_full=use_dataset_full, - clocker_list=clocker_list, - ) - - def object_via_gen_from( - self, fit, instance: Optional[af.ModelInstance] = None - ) -> List[FitDataset1D]: - """ - Returns a generator of `FitDataset1D` objects from an input aggregator. - - See `__init__` for a description of how the `FitDataset1D` objects are created by this method. - - If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible - to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. - - Parameters - ---------- - fit - A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database. - cti - The CTI model used to add CTI to the dataset to perform the fit. - """ - return _fit_dataset_1d_list_from( - fit=fit, - instance=instance, - use_dataset_full=self.use_dataset_full, - clocker_list=self.clocker_list, - ) +from __future__ import annotations +from typing import TYPE_CHECKING, List, Optional + +from autocti.aggregator.abstract import AggBase + +if TYPE_CHECKING: + from autocti.clocker.abstract import AbstractClocker + from autocti.dataset_1d.fit import FitDataset1D + +import autofit as af + +from autocti.aggregator.dataset_1d import _dataset_1d_list_from + + +def _cti_list_from(source, total_datasets: int): + """ + Extract one CTI model per dataset from a model instance. + + A single-analysis instance exposes ``instance.cti`` directly; a factor-graph + instance (multi-dataset fit) is an indexed collection with one child + instance per factor. + """ + if hasattr(source, "cti"): + return [source.cti] * total_datasets + + # A factor-graph instance also carries the FactorGraphModel itself as a + # trailing child, so only children with a CTI model are taken. + cti_list = [child.cti for child in source if hasattr(child, "cti")] + + if len(cti_list) != total_datasets: + raise ValueError( + f"The instance contains {len(cti_list)} CTI models but the fit has " + f"{total_datasets} datasets." + ) + + return cti_list + + +def _fit_dataset_1d_list_from( + fit: af.Fit, + instance: Optional[af.ModelInstance] = None, + use_dataset_full: bool = False, + clocker_list: Optional[AbstractClocker] = None, +) -> List[FitDataset1D]: + """ + Returns a list of `FitDataset1D` object from a `PyAutoFit` sqlite database `Fit` object. + + The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit: + + - The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder). + - The clocker used to add CTI in the fit (`dataset/clocker.json`). + - The settings used for clocking CIT (contained in `dataset/settings_cti.json`). + + Each individual attribute can be loaded from the database via the `fit.value()` method. + + This method combines all of these attributes and returns a list of `FitDataset1D` objects, by loading the masked + dataset adding CTI to its pre-cti data via the cti model and clocking and fitting the model image to the dataset. + + If multiple `Dataset1D` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method + is instead used to load lists of the datasets, perform the fit and return a list of `FitDataset1D` objects. + + If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible + to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. + + Parameters + ---------- + fit + A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database. + instance + A manual instance that overwrites the max log likelihood instance in fit (e.g. for drawing the instance + randomly from the PDF). + use_dataset_full + If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible + to the database, the input `use_dataset_full` can be switched in to load instead the full `Dataset1D` objects. + clocker_list + If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit. + """ + + from autocti.dataset_1d.fit import FitDataset1D + + dataset_list = _dataset_1d_list_from(fit=fit, use_dataset_full=use_dataset_full) + + if clocker_list is None: + if not fit.children: + clocker_list = [fit.value(name="clocker")] + else: + clocker_list = fit.child_values(name="clocker") + + cti_list = _cti_list_from( + source=instance if instance is not None else fit.instance, + total_datasets=len(dataset_list), + ) + + post_cti_data_list = [ + clocker.add_cti(data=dataset.pre_cti_data, cti=cti) + for dataset, clocker, cti in zip(dataset_list, clocker_list, cti_list) + ] + + return [ + FitDataset1D( + dataset=dataset, + post_cti_data=post_cti_data, + ) + for dataset, post_cti_data in zip(dataset_list, post_cti_data_list) + ] + + +class FitDataset1DAgg(AggBase): + def __init__( + self, + aggregator: af.Aggregator, + use_dataset_full: bool = False, + clocker_list: Optional[List[AbstractClocker]] = None, + ): + """ + Interfaces with an `PyAutoFit` aggregator object to create instances of `Dataset1D` objects from the results + of a model-fit. + + The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit: + + - The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder). + - The clocker used to add CTI in the fit (`dataset/clocker.json`). + - The settings used for clocking CIT (contained in `dataset/settings_cti.json`). + + The `aggregator` contains the path to each of these files, and they can be loaded individually. This class + can load them all at once and create a `FitDataset1D` object via the `_fit_dataset_1d_from` method. + + This class's methods returns generators which create the instances of the `FitDataset1D` objects. This ensures + that large sets of results can be efficiently loaded from the hard-disk and do not require storing all + `Dataset1D` instances in the memory at once. + + For example, if the `aggregator` contains 3 model-fits, this class can be used to create a generator which + creates instances of the corresponding 3 `Dataset1D` objects. + + If multiple `Dataset1D` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method + is instead used to load lists of the datasets, perform the fit and return a list of `FitDataset1D` objects. + + If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible + to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. + + This can be done manually, but this object provides a more concise API. + + Parameters + ---------- + aggregator + A `PyAutoFit` aggregator object which can load the results of model-fits. + use_dataset_full + If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore + accessible to the database, the input `use_dataset_full` can be switched in to load instead the + full `Dataset1D` objects. + clocker_list + If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit. + """ + super().__init__( + aggregator=aggregator, + use_dataset_full=use_dataset_full, + clocker_list=clocker_list, + ) + + def object_via_gen_from( + self, fit, instance: Optional[af.ModelInstance] = None + ) -> List[FitDataset1D]: + """ + Returns a generator of `FitDataset1D` objects from an input aggregator. + + See `__init__` for a description of how the `FitDataset1D` objects are created by this method. + + If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible + to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. + + Parameters + ---------- + fit + A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database. + cti + The CTI model used to add CTI to the dataset to perform the fit. + """ + return _fit_dataset_1d_list_from( + fit=fit, + instance=instance, + use_dataset_full=self.use_dataset_full, + clocker_list=self.clocker_list, + ) diff --git a/autocti/aggregator/fit_imaging_ci.py b/autocti/aggregator/fit_imaging_ci.py index 05c036a4..570e2034 100644 --- a/autocti/aggregator/fit_imaging_ci.py +++ b/autocti/aggregator/fit_imaging_ci.py @@ -1,160 +1,184 @@ -from __future__ import annotations -from typing import TYPE_CHECKING, List, Optional, Union - -if TYPE_CHECKING: - from autocti.clocker.abstract import AbstractClocker - from autocti.model.model_util import CTI1D - from autocti.model.model_util import CTI2D - from autocti.charge_injection.fit import FitImagingCI - -import autofit as af - -from autocti.aggregator.abstract import AggBase -from autocti.aggregator.imaging_ci import _imaging_ci_list_from - - -def _fit_imaging_ci_list_from( - fit: af.Fit, - instance: Optional[af.ModelInstance] = None, - use_dataset_full: bool = False, - clocker_list: Optional[AbstractClocker] = None, -) -> List[FitImagingCI]: - """ - Returns a list of `FitImagingCI` object from a `PyAutoFit` sqlite database `Fit` object. - - The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit: - - - The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder). - - The clocker used to add CTI in the fit (`dataset/clocker.json`). - - The settings used for clocking CIT (contained in `dataset/settings_cti.json`). - - Each individual attribute can be loaded from the database via the `fit.value()` method. - - This method combines all of these attributes and returns a list of `FitImagingCI` objects, by loading the masked - dataset adding CTI to its pre-cti data via the cti model and clocking and fitting the model image to the dataset. - - If multiple `ImagingCI` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method - is instead used to load lists of the datasets, perform the fit and return a list of `FitImagingCI` objects. - - If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible - to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. - - Parameters - ---------- - fit - A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database. - instance - A manual instance that overwrites the max log likelihood instance in fit (e.g. for drawing the instance - randomly from the PDF). - use_dataset_full - If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible - to the database, the input `use_dataset_full` can be switched in to load instead the full `ImagingCI` objects. - clocker_list - If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit. - """ - - from autocti.charge_injection.fit import FitImagingCI - - dataset_list = _imaging_ci_list_from(fit=fit, use_dataset_full=use_dataset_full) - - if clocker_list is None: - if not fit.children: - clocker_list = [fit.value(name="clocker")] - else: - clocker_list = fit.child_values(name="clocker") - - if instance is not None: - cti = instance.cti - else: - cti = fit.instance.cti - - post_cti_data_list = [ - clocker.add_cti(data=dataset.pre_cti_data, cti=cti) - for dataset, clocker in zip(dataset_list, clocker_list) - ] - - return [ - FitImagingCI( - dataset=dataset, - post_cti_data=post_cti_data, - ) - for dataset, post_cti_data in zip(dataset_list, post_cti_data_list) - ] - - -class FitImagingCIAgg(AggBase): - def __init__( - self, - aggregator: af.Aggregator, - use_dataset_full: bool = False, - clocker_list: Optional[List[AbstractClocker]] = None, - ): - """ - Interfaces with an `PyAutoFit` aggregator object to create instances of `ImagingCI` objects from the results - of a model-fit. - - The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit: - - - The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder). - - The clocker used to add CTI in the fit (`dataset/clocker.json`). - - The settings used for clocking CIT (contained in `dataset/settings_cti.json`). - - The `aggregator` contains the path to each of these files, and they can be loaded individually. This class - can load them all at once and create a `FitImagingCI` object via the `_fit_dataset_1d_from` method. - - This class's methods returns generators which create the instances of the `FitImagingCI` objects. This ensures - that large sets of results can be efficiently loaded from the hard-disk and do not require storing all - `ImagingCI` instances in the memory at once. - - For example, if the `aggregator` contains 3 model-fits, this class can be used to create a generator which - creates instances of the corresponding 3 `ImagingCI` objects. - - If multiple `ImagingCI` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method - is instead used to load lists of the datasets, perform the fit and return a list of `FitImagingCI` objects. - - If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible - to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. - - This can be done manually, but this object provides a more concise API. - - Parameters - ---------- - aggregator - A `PyAutoFit` aggregator object which can load the results of model-fits. - use_dataset_full - If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore - accessible to the database, the input `use_dataset_full` can be switched in to load instead the - full `ImagingCI` objects. - clocker_list - If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit. - """ - super().__init__( - aggregator=aggregator, - use_dataset_full=use_dataset_full, - clocker_list=clocker_list, - ) - - def object_via_gen_from( - self, fit, instance: Optional[af.ModelInstance] = None - ) -> List[FitImagingCI]: - """ - Returns a generator of `FitImagingCI` objects from an input aggregator. - - See `__init__` for a description of how the `FitImagingCI` objects are created by this method. - - If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible - to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. - - Parameters - ---------- - fit - A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database. - cti - The CTI model used to add CTI to the dataset to perform the fit. - """ - return _fit_imaging_ci_list_from( - fit=fit, - instance=instance, - use_dataset_full=self.use_dataset_full, - clocker_list=self.clocker_list, - ) +from __future__ import annotations +from typing import TYPE_CHECKING, List, Optional, Union + +if TYPE_CHECKING: + from autocti.clocker.abstract import AbstractClocker + from autocti.model.model_util import CTI1D + from autocti.model.model_util import CTI2D + from autocti.charge_injection.fit import FitImagingCI + +import autofit as af + +from autocti.aggregator.abstract import AggBase +from autocti.aggregator.imaging_ci import _imaging_ci_list_from + + +def _cti_list_from(source, total_datasets: int): + """ + Extract one CTI model per dataset from a model instance. + + A single-analysis instance exposes ``instance.cti`` directly; a factor-graph + instance (multi-dataset fit) is an indexed collection with one child + instance per factor. + """ + if hasattr(source, "cti"): + return [source.cti] * total_datasets + + # A factor-graph instance also carries the FactorGraphModel itself as a + # trailing child, so only children with a CTI model are taken. + cti_list = [child.cti for child in source if hasattr(child, "cti")] + + if len(cti_list) != total_datasets: + raise ValueError( + f"The instance contains {len(cti_list)} CTI models but the fit has " + f"{total_datasets} datasets." + ) + + return cti_list + + +def _fit_imaging_ci_list_from( + fit: af.Fit, + instance: Optional[af.ModelInstance] = None, + use_dataset_full: bool = False, + clocker_list: Optional[AbstractClocker] = None, +) -> List[FitImagingCI]: + """ + Returns a list of `FitImagingCI` object from a `PyAutoFit` sqlite database `Fit` object. + + The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit: + + - The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder). + - The clocker used to add CTI in the fit (`dataset/clocker.json`). + - The settings used for clocking CIT (contained in `dataset/settings_cti.json`). + + Each individual attribute can be loaded from the database via the `fit.value()` method. + + This method combines all of these attributes and returns a list of `FitImagingCI` objects, by loading the masked + dataset adding CTI to its pre-cti data via the cti model and clocking and fitting the model image to the dataset. + + If multiple `ImagingCI` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method + is instead used to load lists of the datasets, perform the fit and return a list of `FitImagingCI` objects. + + If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible + to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. + + Parameters + ---------- + fit + A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database. + instance + A manual instance that overwrites the max log likelihood instance in fit (e.g. for drawing the instance + randomly from the PDF). + use_dataset_full + If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible + to the database, the input `use_dataset_full` can be switched in to load instead the full `ImagingCI` objects. + clocker_list + If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit. + """ + + from autocti.charge_injection.fit import FitImagingCI + + dataset_list = _imaging_ci_list_from(fit=fit, use_dataset_full=use_dataset_full) + + if clocker_list is None: + if not fit.children: + clocker_list = [fit.value(name="clocker")] + else: + clocker_list = fit.child_values(name="clocker") + + cti_list = _cti_list_from( + source=instance if instance is not None else fit.instance, + total_datasets=len(dataset_list), + ) + + post_cti_data_list = [ + clocker.add_cti(data=dataset.pre_cti_data, cti=cti) + for dataset, clocker, cti in zip(dataset_list, clocker_list, cti_list) + ] + + return [ + FitImagingCI( + dataset=dataset, + post_cti_data=post_cti_data, + ) + for dataset, post_cti_data in zip(dataset_list, post_cti_data_list) + ] + + +class FitImagingCIAgg(AggBase): + def __init__( + self, + aggregator: af.Aggregator, + use_dataset_full: bool = False, + clocker_list: Optional[List[AbstractClocker]] = None, + ): + """ + Interfaces with an `PyAutoFit` aggregator object to create instances of `ImagingCI` objects from the results + of a model-fit. + + The results of a model-fit can be stored in a sqlite database, including the following attributes of the fit: + + - The masked dataset (e.g. data / noise map / pre cti data) as .fits files (contained in `dataset` folder). + - The clocker used to add CTI in the fit (`dataset/clocker.json`). + - The settings used for clocking CIT (contained in `dataset/settings_cti.json`). + + The `aggregator` contains the path to each of these files, and they can be loaded individually. This class + can load them all at once and create a `FitImagingCI` object via the `_fit_dataset_1d_from` method. + + This class's methods returns generators which create the instances of the `FitImagingCI` objects. This ensures + that large sets of results can be efficiently loaded from the hard-disk and do not require storing all + `ImagingCI` instances in the memory at once. + + For example, if the `aggregator` contains 3 model-fits, this class can be used to create a generator which + creates instances of the corresponding 3 `ImagingCI` objects. + + If multiple `ImagingCI` objects were fitted simultaneously via analysis summing, the `fit.child_values()` method + is instead used to load lists of the datasets, perform the fit and return a list of `FitImagingCI` objects. + + If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible + to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. + + This can be done manually, but this object provides a more concise API. + + Parameters + ---------- + aggregator + A `PyAutoFit` aggregator object which can load the results of model-fits. + use_dataset_full + If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore + accessible to the database, the input `use_dataset_full` can be switched in to load instead the + full `ImagingCI` objects. + clocker_list + If input, overwrites the clocker used in the fit with a new clocker which is used to perform the fit. + """ + super().__init__( + aggregator=aggregator, + use_dataset_full=use_dataset_full, + clocker_list=clocker_list, + ) + + def object_via_gen_from( + self, fit, instance: Optional[af.ModelInstance] = None + ) -> List[FitImagingCI]: + """ + Returns a generator of `FitImagingCI` objects from an input aggregator. + + See `__init__` for a description of how the `FitImagingCI` objects are created by this method. + + If a `dataset_full` is input into the `Analysis` class when a model-fit is performed and therefore accessible + to the database, the input `use_dataset_full` can be switched in to fit the full dataset instead. + + Parameters + ---------- + fit + A `PyAutoFit` `Fit` object which contains the results of a model-fit as an entry in a sqlite database. + cti + The CTI model used to add CTI to the dataset to perform the fit. + """ + return _fit_imaging_ci_list_from( + fit=fit, + instance=instance, + use_dataset_full=self.use_dataset_full, + clocker_list=self.clocker_list, + ) diff --git a/test_autocti/aggregator/conftest.py b/test_autocti/aggregator/conftest.py index 88dbec9c..ece63f8f 100644 --- a/test_autocti/aggregator/conftest.py +++ b/test_autocti/aggregator/conftest.py @@ -1,116 +1,141 @@ -import pytest -from os import path -import os -import shutil - -from autoconf import conf -import autofit as af -import autocti as ac -from autoconf.conf import with_config -from autofit.non_linear.samples import Sample - - -@pytest.fixture(autouse=True) -def set_test_mode(): - os.environ["PYAUTOFIT_TEST_MODE"] = "1" - yield - del os.environ["PYAUTOFIT_TEST_MODE"] - - -def clean(database_file): - database_sqlite = path.join(conf.instance.output_path, f"{database_file}.sqlite") - - if path.exists(database_sqlite): - os.remove(database_sqlite) - - result_path = path.join(conf.instance.output_path, database_file) - - if path.exists(result_path): - shutil.rmtree(result_path) - - -@with_config( - "general", - "output", - "samples_to_csv", - value=True, -) -def aggregator_from(database_file, analysis, model, samples): - result_path = path.join(conf.instance.output_path, database_file) - - clean(database_file=database_file) - - search = ac.m.MockSearch( - samples=samples, result=ac.m.MockResult(model=model, samples=samples) - ) - search.paths = af.DirectoryPaths(path_prefix=database_file) - search.fit(model=model, analysis=analysis) - - database_file = path.join(conf.instance.output_path, f"{database_file}.sqlite") - - agg = af.Aggregator.from_database(filename=database_file) - agg.add_directory(directory=result_path) - - return agg - - -@pytest.fixture(name="model_1d") -def make_model_1d(): - trap_0 = af.Model(ac.TrapInstantCapture) - - trap_list = [trap_0] - - ccd = af.Model(ac.CCDPhase) - - return af.Collection(cti=af.Model(ac.CTI1D, trap_list=trap_list, ccd=ccd)) - - -@pytest.fixture(name="samples_1d") -def make_samples_1d(model_1d): - parameters = [model_1d.prior_count * [1.0], model_1d.prior_count * [10.0]] - - sample_list = Sample.from_lists( - model=model_1d, - parameter_lists=parameters, - log_likelihood_list=[1.0, 2.0], - log_prior_list=[0.0, 0.0], - weight_list=[0.0, 1.0], - ) - - return ac.m.MockSamples( - model=model_1d, - sample_list=sample_list, - prior_means=[1.0] * model_1d.prior_count, - ) - - -@pytest.fixture(name="model_2d") -def make_model_2d(): - trap_0 = af.Model(ac.TrapInstantCapture) - - trap_list = [trap_0] - - ccd = af.Model(ac.CCDPhase) - - return af.Collection( - cti=af.Model(ac.CTI2D, parallel_trap_list=trap_list, parallel_ccd=ccd) - ) - - -@pytest.fixture(name="samples_2d") -def make_samples_2d(model_2d): - parameters = [model_2d.prior_count * [1.0], model_2d.prior_count * [10.0]] - - sample_list = Sample.from_lists( - model=model_2d, - parameter_lists=parameters, - log_likelihood_list=[1.0, 2.0], - log_prior_list=[0.0, 0.0], - weight_list=[0.0, 1.0], - ) - - return ac.m.MockSamples( - model=model_2d, - sample_list=sample_list, - prior_means=[1.0] * model_2d.prior_count, - ) +import pytest +from os import path +import os +import shutil + +from autoconf import conf +import autofit as af +import autocti as ac +from autoconf.conf import with_config +from autofit.non_linear.samples import Sample + + +@pytest.fixture(autouse=True) +def set_test_mode(): + os.environ["PYAUTOFIT_TEST_MODE"] = "1" + yield + del os.environ["PYAUTOFIT_TEST_MODE"] + + +def clean(database_file): + database_sqlite = path.join(conf.instance.output_path, f"{database_file}.sqlite") + + if path.exists(database_sqlite): + os.remove(database_sqlite) + + result_path = path.join(conf.instance.output_path, database_file) + + if path.exists(result_path): + shutil.rmtree(result_path) + + +@with_config( + "general", + "output", + "samples_to_csv", + value=True, +) +def aggregator_from(database_file, analysis, model, samples): + result_path = path.join(conf.instance.output_path, database_file) + + clean(database_file=database_file) + + if isinstance(analysis, (list, tuple)): + # Analysis summing was removed from PyAutoFit; multi-dataset fits are + # expressed as a factor graph with the model shared across factors. + # The samples are rebuilt against the global prior model, whose paths + # carry per-factor prefixes. + factor_list = [ + af.AnalysisFactor(prior_model=model, analysis=analysis_single) + for analysis_single in analysis + ] + analysis = af.FactorGraphModel(*factor_list) + model = analysis.global_prior_model + + sample_list = Sample.from_lists( + model=model, + parameter_lists=samples.parameter_lists, + log_likelihood_list=samples.log_likelihood_list, + log_prior_list=[0.0] * len(samples.log_likelihood_list), + weight_list=samples.weight_list, + ) + samples = ac.m.MockSamples( + model=model, + sample_list=sample_list, + prior_means=[1.0] * model.prior_count, + ) + + search = ac.m.MockSearch( + samples=samples, result=ac.m.MockResult(model=model, samples=samples) + ) + search.paths = af.DirectoryPaths(path_prefix=database_file) + search.fit(model=model, analysis=analysis) + + database_file = path.join(conf.instance.output_path, f"{database_file}.sqlite") + + agg = af.Aggregator.from_database(filename=database_file) + agg.add_directory(directory=result_path) + + return agg + + +@pytest.fixture(name="model_1d") +def make_model_1d(): + trap_0 = af.Model(ac.TrapInstantCapture) + + trap_list = [trap_0] + + ccd = af.Model(ac.CCDPhase) + + return af.Collection(cti=af.Model(ac.CTI1D, trap_list=trap_list, ccd=ccd)) + + +@pytest.fixture(name="samples_1d") +def make_samples_1d(model_1d): + parameters = [model_1d.prior_count * [1.0], model_1d.prior_count * [10.0]] + + sample_list = Sample.from_lists( + model=model_1d, + parameter_lists=parameters, + log_likelihood_list=[1.0, 2.0], + log_prior_list=[0.0, 0.0], + weight_list=[0.0, 1.0], + ) + + return ac.m.MockSamples( + model=model_1d, + sample_list=sample_list, + prior_means=[1.0] * model_1d.prior_count, + ) + + +@pytest.fixture(name="model_2d") +def make_model_2d(): + trap_0 = af.Model(ac.TrapInstantCapture) + + trap_list = [trap_0] + + ccd = af.Model(ac.CCDPhase) + + return af.Collection( + cti=af.Model(ac.CTI2D, parallel_trap_list=trap_list, parallel_ccd=ccd) + ) + + +@pytest.fixture(name="samples_2d") +def make_samples_2d(model_2d): + parameters = [model_2d.prior_count * [1.0], model_2d.prior_count * [10.0]] + + sample_list = Sample.from_lists( + model=model_2d, + parameter_lists=parameters, + log_likelihood_list=[1.0, 2.0], + log_prior_list=[0.0, 0.0], + weight_list=[0.0, 1.0], + ) + + return ac.m.MockSamples( + model=model_2d, + sample_list=sample_list, + prior_means=[1.0] * model_2d.prior_count, + ) diff --git a/test_autocti/aggregator/test_aggregator_dataset_1d.py b/test_autocti/aggregator/test_aggregator_dataset_1d.py index 539d888f..51521ab4 100644 --- a/test_autocti/aggregator/test_aggregator_dataset_1d.py +++ b/test_autocti/aggregator/test_aggregator_dataset_1d.py @@ -32,11 +32,6 @@ def test__dataset_gen_from__analysis_has_single_dataset( clean(database_file=database_file) -@pytest.mark.skip( - reason="Analysis summing (analysis + analysis) was removed from PyAutoFit in favour " - "of AnalysisFactor/FactorGraphModel; these tests are ported in Phase 2 of the CTI " - "resurrection epic (PyAutoCTI#82)." -) def test__dataset_gen_from__analysis_has_multi_dataset( dataset_1d_7, clocker_1d, samples_1d, model_1d ): @@ -44,7 +39,7 @@ def test__dataset_gen_from__analysis_has_multi_dataset( agg = aggregator_from( database_file=database_file, - analysis=analysis + analysis, + analysis=[analysis, analysis], model=model_1d, samples=samples_1d, ) @@ -65,11 +60,6 @@ def test__dataset_gen_from__analysis_has_multi_dataset( clean(database_file=database_file) -@pytest.mark.skip( - reason="Analysis summing (analysis + analysis) was removed from PyAutoFit in favour " - "of AnalysisFactor/FactorGraphModel; these tests are ported in Phase 2 of the CTI " - "resurrection epic (PyAutoCTI#82)." -) def test__dataset_gen_from__analysis_use_dataset_full( dataset_1d_7, clocker_1d, samples_1d, model_1d ): @@ -82,7 +72,7 @@ def test__dataset_gen_from__analysis_use_dataset_full( agg = aggregator_from( database_file=database_file, - analysis=analysis + analysis, + analysis=[analysis, analysis], model=model_1d, samples=samples_1d, ) diff --git a/test_autocti/aggregator/test_aggregator_fit_imaging_ci.py b/test_autocti/aggregator/test_aggregator_fit_imaging_ci.py index 5b46881a..953610c7 100644 --- a/test_autocti/aggregator/test_aggregator_fit_imaging_ci.py +++ b/test_autocti/aggregator/test_aggregator_fit_imaging_ci.py @@ -34,11 +34,6 @@ def test__fit_imaging_ci_randomly_drawn_via_pdf_gen_from( clean(database_file=database_file) -@pytest.mark.skip( - reason="Analysis summing (analysis + analysis) was removed from PyAutoFit in favour " - "of AnalysisFactor/FactorGraphModel; these tests are ported in Phase 2 of the CTI " - "resurrection epic (PyAutoCTI#82)." -) def test__fit_imaging_ci_randomly_drawn_via_pdf_gen_from__multi_analysis( imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d ): @@ -46,7 +41,7 @@ def test__fit_imaging_ci_randomly_drawn_via_pdf_gen_from__multi_analysis( agg = aggregator_from( database_file=database_file, - analysis=analysis + analysis, + analysis=[analysis, analysis], model=model_2d, samples=samples_2d, ) diff --git a/test_autocti/aggregator/test_aggregator_imaging_ci.py b/test_autocti/aggregator/test_aggregator_imaging_ci.py index 8c8454c1..36ecc0f7 100644 --- a/test_autocti/aggregator/test_aggregator_imaging_ci.py +++ b/test_autocti/aggregator/test_aggregator_imaging_ci.py @@ -32,11 +32,6 @@ def test__dataset_gen_from__analysis_has_single_dataset( clean(database_file=database_file) -@pytest.mark.skip( - reason="Analysis summing (analysis + analysis) was removed from PyAutoFit in favour " - "of AnalysisFactor/FactorGraphModel; these tests are ported in Phase 2 of the CTI " - "resurrection epic (PyAutoCTI#82)." -) def test__dataset_gen_from__analysis_has_multi_dataset( imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d ): @@ -44,7 +39,7 @@ def test__dataset_gen_from__analysis_has_multi_dataset( agg = aggregator_from( database_file=database_file, - analysis=analysis + analysis, + analysis=[analysis, analysis], model=model_2d, samples=samples_2d, ) @@ -66,11 +61,6 @@ def test__dataset_gen_from__analysis_has_multi_dataset( clean(database_file=database_file) -@pytest.mark.skip( - reason="Analysis summing (analysis + analysis) was removed from PyAutoFit in favour " - "of AnalysisFactor/FactorGraphModel; these tests are ported in Phase 2 of the CTI " - "resurrection epic (PyAutoCTI#82)." -) def test__dataset_gen_from__analysis_use_dataset_full( imaging_ci_7x7, parallel_clocker_2d, samples_2d, model_2d ): @@ -85,7 +75,7 @@ def test__dataset_gen_from__analysis_use_dataset_full( agg = aggregator_from( database_file=database_file, - analysis=analysis + analysis, + analysis=[analysis, analysis], model=model_2d, samples=samples_2d, )