From 7f3ec55722c6cd0db9359cfa4db934ec3302e7e9 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Tue, 14 Jul 2026 09:15:04 -0400 Subject: [PATCH 1/6] Fixed bug in computing total_sigclip_groups. --- src/stcal/jump/twopoint_difference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/stcal/jump/twopoint_difference.py b/src/stcal/jump/twopoint_difference.py index 952cb4191..6e26f88db 100644 --- a/src/stcal/jump/twopoint_difference.py +++ b/src/stcal/jump/twopoint_difference.py @@ -663,7 +663,7 @@ def groups_all_set_dnu(nints, ngroups, gdq, twopt_p): if twopt_p.only_use_ints: total_sigclip_groups = nints else: - total_sigclip_groups = nints * ngroups - num_flagged_grps + total_sigclip_groups = nints * ngroups - total_flagged_grps min_usable_groups = ngroups - max_flagged_grps total_groups = nints * ngroups - total_flagged_grps From d706da2a78707f5f42563dc5b10b923c76a7c11b Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 15 Jul 2026 10:52:26 -0400 Subject: [PATCH 2/6] Updating jump description. --- docs/stcal/jump/description.rst | 53 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/docs/stcal/jump/description.rst b/docs/stcal/jump/description.rst index bd0bbe7f9..5e295c4ac 100644 --- a/docs/stcal/jump/description.rst +++ b/docs/stcal/jump/description.rst @@ -21,34 +21,31 @@ in `Anderson & Gordon (2011) Date: Wed, 15 Jul 2026 11:30:02 -0400 Subject: [PATCH 3/6] Updating jump description. --- docs/stcal/jump/description.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/stcal/jump/description.rst b/docs/stcal/jump/description.rst index 5e295c4ac..0abacd780 100644 --- a/docs/stcal/jump/description.rst +++ b/docs/stcal/jump/description.rst @@ -36,15 +36,15 @@ then the jump step is skipped. The jump step runs using the first differences of groups in integrations runs as follows. -#. If the groups are evenly spaced in time and A or B is true, then do the following. +#. If the groups are evenly spaced in time and A or B is true, then: #. If A is true, then use ``astropy.stats.sigma_clip`` across integrations, using ``rejection_threshold`` for jump detection. #. If B is true, then use ``astropy.stats.sigma_clip`` across integrations and groups ``rejection_threshold`` for jump detection. -#. Else, do the following. +#. Else: #. If the minimum number of usable groups is greater than ``min_diffs_single_pass+1``. - The ``+1`` is needed because first differences are used; there - which defaults to 10, then detect jumps in each integration using 4 sigma outliers. + The ``+1`` is needed because first differences are used; ``min_diffs_single_pass`` + defaults to 10. Then detect jumps in each integration using 4 sigma outliers. #. Else, do an iterative flagging within each integration. Note that any ramp groups flagged as SATURATED in the input GROUPDQ array From 8e3a3143af5f8e31e82e91bab19a1f7d31572e2b Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 15 Jul 2026 16:28:30 -0400 Subject: [PATCH 4/6] Updating jump detection documentation. --- docs/stcal/jump/description.rst | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/stcal/jump/description.rst b/docs/stcal/jump/description.rst index 0abacd780..e5ec4ff98 100644 --- a/docs/stcal/jump/description.rst +++ b/docs/stcal/jump/description.rst @@ -37,16 +37,29 @@ then the jump step is skipped. The jump step runs using the first differences of groups in integrations runs as follows. #. If the groups are evenly spaced in time and A or B is true, then: + #. If A is true, then use ``astropy.stats.sigma_clip`` across integrations, - using ``rejection_threshold`` for jump detection. + using ``rejection_threshold`` for jump detection. The default value for + ``rejection_threshold`` is 4.0. #. If B is true, then use ``astropy.stats.sigma_clip`` across integrations and groups ``rejection_threshold`` for jump detection. + #. Else: + #. If the minimum number of usable groups is greater than ``min_diffs_single_pass+1``. The ``+1`` is needed because first differences are used; ``min_diffs_single_pass`` - defaults to 10. Then detect jumps in each integration using 4 sigma outliers. + defaults to 10. Then detect jumps in each integration using sigma outliers above + ``rejection_threshold``. #. Else, do an iterative flagging within each integration. +If flagging of the 4 neighbors is requested, then the 4 adjacent pixels will have +ramp jumps flagged in the same group as the central pixel as long as it has a jump +between the min and max requested levels for this option. + +If flagging of groups after a ramp jump is requested, then the groups in the requested +time since a detected ramp jump will be flagged as ramp jumps if the ramp jump is above +the requested threshold. Two thresholds and times are possible for this option. + Note that any ramp groups flagged as SATURATED in the input GROUPDQ array are not used in any of the above calculations and hence will never be marked as containing a jump. From a363d55af4e7b0ff98e7fbbdcc2ddc4453863582 Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Wed, 15 Jul 2026 16:38:15 -0400 Subject: [PATCH 5/6] Adding change fragment. --- changes/562.jump.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/562.jump.rst diff --git a/changes/562.jump.rst b/changes/562.jump.rst new file mode 100644 index 000000000..c063e9f9b --- /dev/null +++ b/changes/562.jump.rst @@ -0,0 +1 @@ +Updated the jump step documentation and fixed a small bug in computing ``total_sigclip_groups``. From 03f37365d08ed300d87bb82b31236d0e522e2fda Mon Sep 17 00:00:00 2001 From: Ken MacDonald Date: Tue, 21 Jul 2026 11:05:24 -0400 Subject: [PATCH 6/6] Revising the jump step description to add back the details of the two-point difference algorithm. --- docs/stcal/jump/description.rst | 93 ++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 36 deletions(-) diff --git a/docs/stcal/jump/description.rst b/docs/stcal/jump/description.rst index e5ec4ff98..6dd404fa1 100644 --- a/docs/stcal/jump/description.rst +++ b/docs/stcal/jump/description.rst @@ -9,56 +9,77 @@ any pixels that have non-positive or NaN values in the gain reference file will have DQ flags "NO_GAIN_VALUE" and "DO_NOT_USE" set in the output PIXELDQ array. The SCI array of the input data is not modified. -Jumps can be detected in two different ways. The primary way is the two-point -difference method described below. The other way is by selecting ``only_use_ints`` -as ``True`` and if there are enough integrations, then ``sigma_clip`` from the -``astropy.stats`` package will be used to detect jumps. The ``sigma_clip`` method -will also be used if the total number of usable groups (number of groups per -integration multiplied by the number of integrations) is above a minimum threshold. +Jumps in the ramps of a given pixel are detected using statistics of the +two-point differences between successive groups. Those statistics either use +sigma clipping or the method described in +`Anderson & Gordon (2011) `_. -The current implementation uses the two-point difference method described -in `Anderson & Gordon (2011) `_. - -Two-Point Difference Method -^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The two-point difference method is applied as follows: - -It is based on two conditions: +The flow control of the jump step is based on two conditions: A. If ``only_use_ints`` is ``True``, then the number of integrations needs to be greater than ``minimum_sigclip_groups``, which has a default of 100. -B. If ``only_use_ints`` is ``False``, then the total number of available groupsi +B. If ``only_use_ints`` is ``False``, then the total number of available groups needs to be greater than ``minimum_sigclip_groups``. If both A and B are false and there are not enough usable groups in each integration, -then the jump step is skipped. +``minimum_groups`` defaults to 3, then the jump step is skipped. -The jump step runs using the first differences of groups in integrations runs as follows. +If A is ``True``, then sigma clip across integrations for each group difference +in the ramp (e.g., sigma clip groups 3-2 for all integrations, then sigma clip +groups 4-3 for all integrations, etc). -#. If the groups are evenly spaced in time and A or B is true, then: +If B is ``True``, then sigma clip across all group differences and all integrations +simultaneously (e.g., treat all group differences within an integration and in other +integrations equally). - #. If A is true, then use ``astropy.stats.sigma_clip`` across integrations, - using ``rejection_threshold`` for jump detection. The default value for - ``rejection_threshold`` is 4.0. - #. If B is true, then use ``astropy.stats.sigma_clip`` across integrations - and groups ``rejection_threshold`` for jump detection. +If neither A, nor B, are ``True``, then ``astropy.stats.sigma_clip`` is not used. +The algorithm detailed below is used on groups within each integration or is used +over all groups in all integrations. -#. Else: +If there are at least ``min_diffs_single_pass`` in each integration, then use the +two-point difference detailed below over the first group differences in each +integration. This will do a single pass over the first group differences in each +integration. - #. If the minimum number of usable groups is greater than ``min_diffs_single_pass+1``. - The ``+1`` is needed because first differences are used; ``min_diffs_single_pass`` - defaults to 10. Then detect jumps in each integration using sigma outliers above - ``rejection_threshold``. - #. Else, do an iterative flagging within each integration. +Otherwise, use the two-point difference detailed below looping over all groups +across all integrations. This will be an iterative approach that loops over all +first group differences, :math:`(ngroups-1) * nints`, where :math:`ngroups` is +the number of groups in each integration (the :math:`-1` is used because the +operations are on the first differences) and :math:`nints` is the number of +integrations. -If flagging of the 4 neighbors is requested, then the 4 adjacent pixels will have -ramp jumps flagged in the same group as the central pixel as long as it has a jump -between the min and max requested levels for this option. - -If flagging of groups after a ramp jump is requested, then the groups in the requested -time since a detected ramp jump will be flagged as ramp jumps if the ramp jump is above -the requested threshold. Two thresholds and times are possible for this option. +Two-Point Difference Method +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The two-point difference method is applied to each integration as follows: + +#. Compute the first differences for each pixel (the difference between + adjacent groups) +#. Compute the clipped median (dropping the largest difference) of the first + differences for each pixel. If there are only three first difference values + (four groups), no clipping is performed when computing the median. +#. Use the median to estimate the Poisson noise for each group and combine it + with the read noise to arrive at an estimate of the total expected noise for + each difference. +#. Compute the "difference ratio" as the difference between the first differences + of each group and the median, divided by the expected noise. +#. If the largest "difference ratio" is greater than the rejection threshold, + flag the group corresponding to that ratio as having a jump. +#. If a jump is found in a given pixel, iterate the above steps with the + jump-impacted group excluded, looking for additional lower-level jumps + that still exceed the rejection threshold. +#. Stop iterating on a given pixel when no new jumps are found or only one + difference remains. +#. If there are only two differences (three groups), the smallest one is compared + to the larger one and if the larger one is above a threshold, it is flagged + as a jump. +#. If flagging of the 4 neighbors is requested, then the 4 adjacent pixels will + have ramp jumps flagged in the same group as the central pixel as long as it has + a jump between the min and max requested levels for this option. +#. If flagging of groups after a ramp jump is requested, then the groups in the + requested time since a detected ramp jump will be flagged as ramp jumps if + the ramp jump is above the requested threshold. Two thresholds and times are + possible for this option. Note that any ramp groups flagged as SATURATED in the input GROUPDQ array are not used in any of the above calculations and hence will never be