From 6571cfc8eb89a4bfdce355e710a8f7816a123eb7 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Wed, 24 Jun 2026 15:41:04 -0700 Subject: [PATCH 01/21] feat(nimbus): add weekly Celery task to update holdback enrollment period and trigger Jetstream rerun --- experimenter/experimenter/jetstream/tasks.py | 49 ++++++++++++++++++++ experimenter/experimenter/settings.py | 4 ++ 2 files changed, 53 insertions(+) diff --git a/experimenter/experimenter/jetstream/tasks.py b/experimenter/experimenter/jetstream/tasks.py index 2dc1b03967..7615e05ff0 100644 --- a/experimenter/experimenter/jetstream/tasks.py +++ b/experimenter/experimenter/jetstream/tasks.py @@ -196,3 +196,52 @@ def fetch_monitoring_data(): metrics.incr("fetch_monitoring_data.failed") logger.exception(f"Fatal error in fetch_monitoring_data task: {e}") raise + + +@app.task +@metrics.timer_decorator("update_holdback_enrollment_period") +def update_holdback_enrollment_period(): + metrics.incr("update_holdback_enrollment_period.started") + try: + today = timezone.now().date() + now = timezone.now() + + experiments = NimbusExperiment.objects.filter( + is_holdback=True, + status=NimbusExperiment.Status.LIVE, + ).exclude(_start_date=None) + + updated_count = 0 + for experiment in experiments: + enrollment_end = experiment.actual_enrollment_end_date + if enrollment_end is None or today <= enrollment_end: + continue + + days_since_end = (today - enrollment_end).days + weeks_elapsed = max(1, days_since_end // 7) + new_enrollment_period = min( + experiment.proposed_enrollment + (weeks_elapsed * 7), + experiment.proposed_duration, + ) + + experiment.proposed_enrollment = new_enrollment_period + experiment.do_rerun = True + experiment.do_rerun_timestamp = now + experiment.save( + update_fields=[ + "proposed_enrollment", + "do_rerun", + "do_rerun_timestamp", + ] + ) + updated_count += 1 + + logger.info( + f"update_holdback_enrollment_period: updated {updated_count} experiments" + ) + metrics.incr("update_holdback_enrollment_period.completed") + + except Exception as e: + metrics.incr("update_holdback_enrollment_period.failed") + logger.exception(f"Fatal error in update_holdback_enrollment_period: {e}") + raise diff --git a/experimenter/experimenter/settings.py b/experimenter/experimenter/settings.py index 9c90f8e08b..03d6b03f9f 100644 --- a/experimenter/experimenter/settings.py +++ b/experimenter/experimenter/settings.py @@ -421,6 +421,10 @@ "task": "experimenter.experiments.tasks.warm_api_caches", "schedule": config("API_CACHE_WARMING_INTERVAL", default=3600, cast=int), }, + "update_holdback_enrollment_period": { + "task": "experimenter.jetstream.tasks.update_holdback_enrollment_period", + "schedule": crontab(minute=0, hour=7, day_of_week=1), + }, } CELERY_TASK_ROUTES = { "experimenter.kinto.tasks.*": {"queue": "remote_settings"}, From 0729b504bb3c2920f4d3c7a5d8dc6708ae9a3eb8 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Wed, 24 Jun 2026 15:49:08 -0700 Subject: [PATCH 02/21] test(nimbus): add tests for update_holdback_enrollment_period task --- .../jetstream/tests/test_tasks.py | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/experimenter/experimenter/jetstream/tests/test_tasks.py b/experimenter/experimenter/jetstream/tests/test_tasks.py index a7a9504626..a32c06a4e9 100644 --- a/experimenter/experimenter/jetstream/tests/test_tasks.py +++ b/experimenter/experimenter/jetstream/tests/test_tasks.py @@ -3949,3 +3949,82 @@ def test_result_is_cached(self): get_featmon_slugs() get_featmon_slugs() mock_read.assert_called_once() + + +class TestUpdateHoldbackEnrollmentPeriod(TestCase): + def _make_holdback(self, start_date, enrollment_days, duration_days, end_date=None): + experiment = NimbusExperimentFactory.create_with_lifecycle( + NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, + is_holdback=True, + _start_date=start_date, + proposed_enrollment=enrollment_days, + proposed_duration=duration_days, + ) + if end_date: + experiment._computed_end_date = end_date + experiment.save(update_fields=["_computed_end_date"]) + return experiment + + def test_updates_enrollment_period_and_sets_do_rerun(self): + today = datetime.date.today() + start = today - datetime.timedelta(days=50) + # enrollment ended 22 days ago (14-day enrollment period started at start) + experiment = NimbusExperimentFactory.create_with_lifecycle( + NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, + is_holdback=True, + _start_date=start, + proposed_enrollment=14, + proposed_duration=84, + ) + tasks.update_holdback_enrollment_period() + experiment.refresh_from_db() + + # weeks_elapsed = 22 // 7 = 3 → 14 + 21 = 35 + self.assertEqual(experiment.proposed_enrollment, 35) + self.assertTrue(experiment.do_rerun) + self.assertIsNotNone(experiment.do_rerun_timestamp) + + def test_skips_experiment_still_in_enrollment(self): + today = datetime.date.today() + experiment = NimbusExperimentFactory.create_with_lifecycle( + NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, + is_holdback=True, + _start_date=today - datetime.timedelta(days=5), + proposed_enrollment=14, + proposed_duration=84, + ) + original_enrollment = experiment.proposed_enrollment + tasks.update_holdback_enrollment_period() + experiment.refresh_from_db() + + self.assertEqual(experiment.proposed_enrollment, original_enrollment) + self.assertFalse(experiment.do_rerun) + + def test_skips_non_holdback_experiments(self): + today = datetime.date.today() + experiment = NimbusExperimentFactory.create_with_lifecycle( + NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, + is_holdback=False, + _start_date=today - datetime.timedelta(days=50), + proposed_enrollment=14, + proposed_duration=84, + ) + tasks.update_holdback_enrollment_period() + experiment.refresh_from_db() + + self.assertFalse(experiment.do_rerun) + + def test_caps_enrollment_at_proposed_duration(self): + today = datetime.date.today() + experiment = NimbusExperimentFactory.create_with_lifecycle( + NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, + is_holdback=True, + _start_date=today - datetime.timedelta(days=200), + proposed_enrollment=14, + proposed_duration=84, + ) + tasks.update_holdback_enrollment_period() + experiment.refresh_from_db() + + self.assertEqual(experiment.proposed_enrollment, 84) + self.assertTrue(experiment.do_rerun) From e85d07631867e4e94f51b88ef9ffddac4b91ea66 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Thu, 25 Jun 2026 08:48:07 -0700 Subject: [PATCH 03/21] fix(nimbus): use computed_enrollment_end_date in holdback task and fix test expectations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Holdback experiments never pause enrollment so actual_enrollment_end_date is always None, causing the task to skip every experiment. Switch to computed_enrollment_end_date (start_date + proposed_enrollment) which correctly reflects when the initial enrollment window expires. Update test comment and expected value to match the corrected calculation (36 days since end → weeks_elapsed=5 → 14+35=49). --- experimenter/experimenter/jetstream/tasks.py | 2 +- experimenter/experimenter/jetstream/tests/test_tasks.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/experimenter/experimenter/jetstream/tasks.py b/experimenter/experimenter/jetstream/tasks.py index 7615e05ff0..5078a4a7ba 100644 --- a/experimenter/experimenter/jetstream/tasks.py +++ b/experimenter/experimenter/jetstream/tasks.py @@ -213,7 +213,7 @@ def update_holdback_enrollment_period(): updated_count = 0 for experiment in experiments: - enrollment_end = experiment.actual_enrollment_end_date + enrollment_end = experiment.computed_enrollment_end_date if enrollment_end is None or today <= enrollment_end: continue diff --git a/experimenter/experimenter/jetstream/tests/test_tasks.py b/experimenter/experimenter/jetstream/tests/test_tasks.py index a32c06a4e9..635bc6c90c 100644 --- a/experimenter/experimenter/jetstream/tests/test_tasks.py +++ b/experimenter/experimenter/jetstream/tests/test_tasks.py @@ -3968,7 +3968,7 @@ def _make_holdback(self, start_date, enrollment_days, duration_days, end_date=No def test_updates_enrollment_period_and_sets_do_rerun(self): today = datetime.date.today() start = today - datetime.timedelta(days=50) - # enrollment ended 22 days ago (14-day enrollment period started at start) + # enrollment ended 36 days ago (14-day enrollment period started at start) experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, is_holdback=True, @@ -3979,8 +3979,8 @@ def test_updates_enrollment_period_and_sets_do_rerun(self): tasks.update_holdback_enrollment_period() experiment.refresh_from_db() - # weeks_elapsed = 22 // 7 = 3 → 14 + 21 = 35 - self.assertEqual(experiment.proposed_enrollment, 35) + # weeks_elapsed = 36 // 7 = 5 → 14 + 35 = 49 + self.assertEqual(experiment.proposed_enrollment, 49) self.assertTrue(experiment.do_rerun) self.assertIsNotNone(experiment.do_rerun_timestamp) From 932752b4fab55615fde1dacc85a18f34fa5d6d7b Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Mon, 29 Jun 2026 09:58:25 -0700 Subject: [PATCH 04/21] test(nimbus): add error path coverage for update_holdback_enrollment_period --- experimenter/experimenter/jetstream/tests/test_tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/experimenter/experimenter/jetstream/tests/test_tasks.py b/experimenter/experimenter/jetstream/tests/test_tasks.py index 635bc6c90c..a4a6890042 100644 --- a/experimenter/experimenter/jetstream/tests/test_tasks.py +++ b/experimenter/experimenter/jetstream/tests/test_tasks.py @@ -4028,3 +4028,10 @@ def test_caps_enrollment_at_proposed_duration(self): self.assertEqual(experiment.proposed_enrollment, 84) self.assertTrue(experiment.do_rerun) + + def test_raises_on_unexpected_error(self): + with patch( + "experimenter.jetstream.tasks.NimbusExperiment.objects.filter", + side_effect=Exception("db error"), + ), self.assertRaises(Exception, msg="db error"): + tasks.update_holdback_enrollment_period() From d286993f53beb6a44bf3e293b534f869d7945957 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Mon, 29 Jun 2026 11:02:14 -0700 Subject: [PATCH 05/21] style(nimbus): reformat test_raises_on_unexpected_error with parenthesized context managers --- .../experimenter/jetstream/tests/test_tasks.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/experimenter/experimenter/jetstream/tests/test_tasks.py b/experimenter/experimenter/jetstream/tests/test_tasks.py index a4a6890042..d92a08e5f3 100644 --- a/experimenter/experimenter/jetstream/tests/test_tasks.py +++ b/experimenter/experimenter/jetstream/tests/test_tasks.py @@ -4030,8 +4030,11 @@ def test_caps_enrollment_at_proposed_duration(self): self.assertTrue(experiment.do_rerun) def test_raises_on_unexpected_error(self): - with patch( - "experimenter.jetstream.tasks.NimbusExperiment.objects.filter", - side_effect=Exception("db error"), - ), self.assertRaises(Exception, msg="db error"): + with ( + patch( + "experimenter.jetstream.tasks.NimbusExperiment.objects.filter", + side_effect=Exception("db error"), + ), + self.assertRaises(Exception, msg="db error"), + ): tasks.update_holdback_enrollment_period() From 3f31ba2707e9f52993f51197c902980c0c163ed8 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Tue, 30 Jun 2026 15:44:01 -0700 Subject: [PATCH 06/21] fix(nimbus): rework holdback task and serializer based on review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Run daily instead of weekly (holdbacks can launch any day) - Use today - 21 days as rolling enrollment_end (observation period) - Skip experiments with end_date set (actually ended holdbacks) - Update _enrollment_end_date instead of proposed_enrollment - Add changelog entry per updated experiment - Compute proposedEnrollment, enrollmentEndDate, endDate on-the-fly in v8 serializer for active holdbacks — real fields stay untouched - Add HOLDBACK_ENROLLMENT_UPDATED changelog message constant --- .../experiments/api/v8/serializers.py | 27 ++++++++++-- .../experimenter/experiments/models.py | 1 + experimenter/experimenter/jetstream/tasks.py | 24 ++++++----- .../jetstream/tests/test_tasks.py | 43 +++++++------------ experimenter/experimenter/settings.py | 2 +- 5 files changed, 54 insertions(+), 43 deletions(-) diff --git a/experimenter/experimenter/experiments/api/v8/serializers.py b/experimenter/experimenter/experiments/api/v8/serializers.py index a15ae6f19a..ad8a63c5f6 100644 --- a/experimenter/experimenter/experiments/api/v8/serializers.py +++ b/experimenter/experimenter/experiments/api/v8/serializers.py @@ -1,4 +1,5 @@ import contextlib +import datetime import json from django.conf import settings @@ -106,10 +107,10 @@ class NimbusExperimentSerializer(serializers.ModelSerializer): outcomes = serializers.SerializerMethodField() segments = serializers.SerializerMethodField() startDate = serializers.DateField(source="start_date") - enrollmentEndDate = serializers.DateField(source="actual_enrollment_end_date") - endDate = serializers.DateField(source="end_date") + enrollmentEndDate = serializers.SerializerMethodField() + endDate = serializers.SerializerMethodField() proposedDuration = serializers.ReadOnlyField(source="proposed_duration") - proposedEnrollment = serializers.ReadOnlyField(source="proposed_enrollment") + proposedEnrollment = serializers.SerializerMethodField() referenceBranch = serializers.SerializerMethodField() featureValidationOptOut = serializers.ReadOnlyField( source="is_client_schema_disabled" @@ -168,6 +169,26 @@ class Meta: "requiresRestart", ) + def get_enrollmentEndDate(self, obj): + if obj.is_holdback and not obj._end_date and obj._enrollment_end_date: + return obj._enrollment_end_date + return obj.actual_enrollment_end_date + + def get_endDate(self, obj): + if obj.is_holdback and not obj._end_date and obj._enrollment_end_date: + return obj._enrollment_end_date + datetime.timedelta(days=21) + return obj.end_date + + def get_proposedEnrollment(self, obj): + if ( + obj.is_holdback + and not obj.end_date + and obj._enrollment_end_date + and obj.start_date + ): + return (obj._enrollment_end_date - obj.start_date).days + return obj.proposed_enrollment + def get_application(self, obj): return self.get_appId(obj) diff --git a/experimenter/experimenter/experiments/models.py b/experimenter/experimenter/experiments/models.py index 75af68d3ff..547c7ef3fd 100644 --- a/experimenter/experimenter/experiments/models.py +++ b/experimenter/experimenter/experiments/models.py @@ -3286,6 +3286,7 @@ class Messages: COMPLETED = "Experiment is complete" RESULTS_UPDATED = "Experiment results updated" MONITORING_DATA_UPDATED = "Experiment monitoring data updated" + HOLDBACK_ENROLLMENT_UPDATED = "Holdback enrollment period updated" EXPIRED_FROM_PREVIEW = "Expired from preview collection after 30 days" REMOVED_FROM_PREVIEW = "Removed from preview collection" PUSHED_TO_PREVIEW = "Pushed to preview collection" diff --git a/experimenter/experimenter/jetstream/tasks.py b/experimenter/experimenter/jetstream/tasks.py index 5078a4a7ba..d13d4f9c93 100644 --- a/experimenter/experimenter/jetstream/tasks.py +++ b/experimenter/experimenter/jetstream/tasks.py @@ -198,6 +198,9 @@ def fetch_monitoring_data(): raise +HOLDBACK_OBSERVATION_DAYS = 21 + + @app.task @metrics.timer_decorator("update_holdback_enrollment_period") def update_holdback_enrollment_period(): @@ -205,35 +208,34 @@ def update_holdback_enrollment_period(): try: today = timezone.now().date() now = timezone.now() + enrollment_end = today - dt.timedelta(days=HOLDBACK_OBSERVATION_DAYS) experiments = NimbusExperiment.objects.filter( is_holdback=True, status=NimbusExperiment.Status.LIVE, + _end_date=None, ).exclude(_start_date=None) updated_count = 0 for experiment in experiments: - enrollment_end = experiment.computed_enrollment_end_date - if enrollment_end is None or today <= enrollment_end: + if enrollment_end <= experiment.start_date: continue - days_since_end = (today - enrollment_end).days - weeks_elapsed = max(1, days_since_end // 7) - new_enrollment_period = min( - experiment.proposed_enrollment + (weeks_elapsed * 7), - experiment.proposed_duration, - ) - - experiment.proposed_enrollment = new_enrollment_period + experiment._enrollment_end_date = enrollment_end experiment.do_rerun = True experiment.do_rerun_timestamp = now experiment.save( update_fields=[ - "proposed_enrollment", + "_enrollment_end_date", "do_rerun", "do_rerun_timestamp", ] ) + generate_nimbus_changelog( + experiment, + get_kinto_user(), + message=NimbusChangeLog.Messages.HOLDBACK_ENROLLMENT_UPDATED, + ) updated_count += 1 logger.info( diff --git a/experimenter/experimenter/jetstream/tests/test_tasks.py b/experimenter/experimenter/jetstream/tests/test_tasks.py index d92a08e5f3..282d222cf0 100644 --- a/experimenter/experimenter/jetstream/tests/test_tasks.py +++ b/experimenter/experimenter/jetstream/tests/test_tasks.py @@ -3952,23 +3952,9 @@ def test_result_is_cached(self): class TestUpdateHoldbackEnrollmentPeriod(TestCase): - def _make_holdback(self, start_date, enrollment_days, duration_days, end_date=None): - experiment = NimbusExperimentFactory.create_with_lifecycle( - NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, - is_holdback=True, - _start_date=start_date, - proposed_enrollment=enrollment_days, - proposed_duration=duration_days, - ) - if end_date: - experiment._computed_end_date = end_date - experiment.save(update_fields=["_computed_end_date"]) - return experiment - - def test_updates_enrollment_period_and_sets_do_rerun(self): + def test_sets_enrollment_end_date_and_do_rerun(self): today = datetime.date.today() start = today - datetime.timedelta(days=50) - # enrollment ended 36 days ago (14-day enrollment period started at start) experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, is_holdback=True, @@ -3979,55 +3965,56 @@ def test_updates_enrollment_period_and_sets_do_rerun(self): tasks.update_holdback_enrollment_period() experiment.refresh_from_db() - # weeks_elapsed = 36 // 7 = 5 → 14 + 35 = 49 - self.assertEqual(experiment.proposed_enrollment, 49) + expected_enrollment_end = today - datetime.timedelta(days=21) + self.assertEqual(experiment._enrollment_end_date, expected_enrollment_end) self.assertTrue(experiment.do_rerun) self.assertIsNotNone(experiment.do_rerun_timestamp) - def test_skips_experiment_still_in_enrollment(self): + def test_skips_experiment_started_within_observation_period(self): today = datetime.date.today() experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, is_holdback=True, - _start_date=today - datetime.timedelta(days=5), + _start_date=today - datetime.timedelta(days=10), proposed_enrollment=14, proposed_duration=84, ) - original_enrollment = experiment.proposed_enrollment tasks.update_holdback_enrollment_period() experiment.refresh_from_db() - self.assertEqual(experiment.proposed_enrollment, original_enrollment) + self.assertIsNone(experiment._enrollment_end_date) self.assertFalse(experiment.do_rerun) - def test_skips_non_holdback_experiments(self): + def test_skips_ended_holdback(self): today = datetime.date.today() experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, - is_holdback=False, + is_holdback=True, _start_date=today - datetime.timedelta(days=50), proposed_enrollment=14, proposed_duration=84, ) + experiment._end_date = today - datetime.timedelta(days=1) + experiment.save(update_fields=["_end_date"]) tasks.update_holdback_enrollment_period() experiment.refresh_from_db() + self.assertIsNone(experiment._enrollment_end_date) self.assertFalse(experiment.do_rerun) - def test_caps_enrollment_at_proposed_duration(self): + def test_skips_non_holdback_experiments(self): today = datetime.date.today() experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, - is_holdback=True, - _start_date=today - datetime.timedelta(days=200), + is_holdback=False, + _start_date=today - datetime.timedelta(days=50), proposed_enrollment=14, proposed_duration=84, ) tasks.update_holdback_enrollment_period() experiment.refresh_from_db() - self.assertEqual(experiment.proposed_enrollment, 84) - self.assertTrue(experiment.do_rerun) + self.assertFalse(experiment.do_rerun) def test_raises_on_unexpected_error(self): with ( diff --git a/experimenter/experimenter/settings.py b/experimenter/experimenter/settings.py index 03d6b03f9f..54d073eabb 100644 --- a/experimenter/experimenter/settings.py +++ b/experimenter/experimenter/settings.py @@ -423,7 +423,7 @@ }, "update_holdback_enrollment_period": { "task": "experimenter.jetstream.tasks.update_holdback_enrollment_period", - "schedule": crontab(minute=0, hour=7, day_of_week=1), + "schedule": crontab(minute=0, hour=7), }, } CELERY_TASK_ROUTES = { From c83d3b8bbaf2cb9b6f8723145b74aafcd80c2589 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Tue, 30 Jun 2026 16:00:58 -0700 Subject: [PATCH 07/21] chore(docs): update OpenAPI schema and Swagger UI --- docs/experimenter/openapi-schema.json | 6 ++---- docs/experimenter/swagger-ui.html | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/docs/experimenter/openapi-schema.json b/docs/experimenter/openapi-schema.json index 6e4ad500b3..482f543cfe 100644 --- a/docs/experimenter/openapi-schema.json +++ b/docs/experimenter/openapi-schema.json @@ -1647,11 +1647,11 @@ }, "enrollmentEndDate": { "type": "string", - "format": "date" + "readOnly": true }, "endDate": { "type": "string", - "format": "date" + "readOnly": true }, "proposedDuration": { "type": "string", @@ -1711,8 +1711,6 @@ "channel", "bucketConfig", "startDate", - "enrollmentEndDate", - "endDate", "publishedDate" ] }, diff --git a/docs/experimenter/swagger-ui.html b/docs/experimenter/swagger-ui.html index 10cc5e59bd..0fda984741 100644 --- a/docs/experimenter/swagger-ui.html +++ b/docs/experimenter/swagger-ui.html @@ -1659,11 +1659,11 @@ }, "enrollmentEndDate": { "type": "string", - "format": "date" + "readOnly": true }, "endDate": { "type": "string", - "format": "date" + "readOnly": true }, "proposedDuration": { "type": "string", @@ -1723,8 +1723,6 @@ "channel", "bucketConfig", "startDate", - "enrollmentEndDate", - "endDate", "publishedDate" ] }, From 5ae8bf21de39dca5409c30225c936695c881f9bf Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Tue, 30 Jun 2026 16:03:26 -0700 Subject: [PATCH 08/21] fix(nimbus): use queryset update to avoid protected attribute access on _enrollment_end_date --- experimenter/experimenter/jetstream/tasks.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/experimenter/experimenter/jetstream/tasks.py b/experimenter/experimenter/jetstream/tasks.py index d13d4f9c93..31f83bc113 100644 --- a/experimenter/experimenter/jetstream/tasks.py +++ b/experimenter/experimenter/jetstream/tasks.py @@ -221,16 +221,12 @@ def update_holdback_enrollment_period(): if enrollment_end <= experiment.start_date: continue - experiment._enrollment_end_date = enrollment_end - experiment.do_rerun = True - experiment.do_rerun_timestamp = now - experiment.save( - update_fields=[ - "_enrollment_end_date", - "do_rerun", - "do_rerun_timestamp", - ] + NimbusExperiment.objects.filter(pk=experiment.pk).update( + _enrollment_end_date=enrollment_end, + do_rerun=True, + do_rerun_timestamp=now, ) + experiment.refresh_from_db() generate_nimbus_changelog( experiment, get_kinto_user(), From 5e9abd2256a44a7abf4524451b3610e065656cec Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Tue, 30 Jun 2026 16:07:32 -0700 Subject: [PATCH 09/21] fix(nimbus): use public properties and ISO format strings in holdback serializer methods --- .../experiments/api/v8/serializers.py | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/experimenter/experimenter/experiments/api/v8/serializers.py b/experimenter/experimenter/experiments/api/v8/serializers.py index ad8a63c5f6..94af90da40 100644 --- a/experimenter/experimenter/experiments/api/v8/serializers.py +++ b/experimenter/experimenter/experiments/api/v8/serializers.py @@ -170,23 +170,21 @@ class Meta: ) def get_enrollmentEndDate(self, obj): - if obj.is_holdback and not obj._end_date and obj._enrollment_end_date: - return obj._enrollment_end_date - return obj.actual_enrollment_end_date + enrollment_end = obj.actual_enrollment_end_date + if obj.is_holdback and not obj.end_date and enrollment_end: + return enrollment_end.isoformat() + return enrollment_end.isoformat() if enrollment_end else None def get_endDate(self, obj): - if obj.is_holdback and not obj._end_date and obj._enrollment_end_date: - return obj._enrollment_end_date + datetime.timedelta(days=21) - return obj.end_date + enrollment_end = obj.actual_enrollment_end_date + if obj.is_holdback and not obj.end_date and enrollment_end: + return (enrollment_end + datetime.timedelta(days=21)).isoformat() + return obj.end_date.isoformat() if obj.end_date else None def get_proposedEnrollment(self, obj): - if ( - obj.is_holdback - and not obj.end_date - and obj._enrollment_end_date - and obj.start_date - ): - return (obj._enrollment_end_date - obj.start_date).days + enrollment_end = obj.actual_enrollment_end_date + if obj.is_holdback and not obj.end_date and enrollment_end and obj.start_date: + return (enrollment_end - obj.start_date).days return obj.proposed_enrollment def get_application(self, obj): From 593685628c39cc8ae330e4aec922e2293efe3fe1 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Tue, 30 Jun 2026 16:12:09 -0700 Subject: [PATCH 10/21] test(nimbus): add coverage for holdback serializer method overrides --- .../tests/api/v8/test_serializers.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py b/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py index 1656fb8682..8b76e09cbc 100644 --- a/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py +++ b/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py @@ -419,6 +419,29 @@ def test_localized_localizations_json(self, l10n_json, expected): else: self.assertEqual(serializer.data["localizations"], expected) + def test_holdback_serializer_overrides(self): + today = datetime.date.today() + start = today - datetime.timedelta(days=50) + enrollment_end = today - datetime.timedelta(days=21) + experiment = NimbusExperimentFactory.create_with_lifecycle( + NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, + is_holdback=True, + _start_date=start, + _enrollment_end_date=enrollment_end, + ) + serializer = NimbusExperimentSerializer(experiment) + data = serializer.data + + self.assertEqual(data["enrollmentEndDate"], enrollment_end.isoformat()) + self.assertEqual( + data["endDate"], + (enrollment_end + datetime.timedelta(days=21)).isoformat(), + ) + self.assertEqual( + data["proposedEnrollment"], + (enrollment_end - start).days, + ) + def _experiment_data_without_branches_and_featureIds( self, experiment_data, min_required_version ) -> dict[str, Any]: From 01c4729302936767de927cc73bd8d9e89b07f524 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Tue, 7 Jul 2026 11:21:06 -0700 Subject: [PATCH 11/21] test(nimbus): add coverage for non-holdback proposedEnrollment fallback path --- .../experiments/tests/api/v8/test_serializers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py b/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py index a3ecdce9f3..f66d40fb50 100644 --- a/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py +++ b/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py @@ -439,6 +439,15 @@ def test_holdback_serializer_overrides(self): (enrollment_end - start).days, ) + def test_non_holdback_proposed_enrollment_uses_model_value(self): + experiment = NimbusExperimentFactory.create_with_lifecycle( + NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, + is_holdback=False, + proposed_enrollment=14, + ) + serializer = NimbusExperimentSerializer(experiment) + self.assertEqual(serializer.data["proposedEnrollment"], 14) + def _experiment_data_without_branches_and_featureIds( self, experiment_data, min_required_version ) -> dict[str, Any]: From 72fe48bee044b780d71113f296ab5f707159ee6a Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Tue, 7 Jul 2026 16:10:43 -0700 Subject: [PATCH 12/21] fix(nimbus): address review feedback on holdback task and serializer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move HOLDBACK_OBSERVATION_DAYS constant to settings.py - Remove _enrollment_end_date update from task — serializer computes on-the-fly - Serializer computes fake enrollment end (today - 21 days) when holdback has no explicit enrollment_end_date, rather than relying on stored value - Use shared constant from settings in both task and serializer --- .../experiments/api/v8/serializers.py | 27 +++++++++++++------ .../tests/api/v8/test_serializers.py | 12 +++------ experimenter/experimenter/jetstream/tasks.py | 6 +---- .../jetstream/tests/test_tasks.py | 7 ++--- experimenter/experimenter/settings.py | 2 ++ 5 files changed, 28 insertions(+), 26 deletions(-) diff --git a/experimenter/experimenter/experiments/api/v8/serializers.py b/experimenter/experimenter/experiments/api/v8/serializers.py index 39de51441d..030f7bc77c 100644 --- a/experimenter/experimenter/experiments/api/v8/serializers.py +++ b/experimenter/experimenter/experiments/api/v8/serializers.py @@ -162,22 +162,33 @@ class Meta: "requiresRestart", ) + def _holdback_enrollment_end(self, obj): + if obj.is_holdback and not obj.end_date and not obj.actual_enrollment_end_date: + return datetime.date.today() - datetime.timedelta( + days=settings.HOLDBACK_OBSERVATION_DAYS + ) + return None + def get_enrollmentEndDate(self, obj): + holdback_end = self._holdback_enrollment_end(obj) + if holdback_end: + return holdback_end.isoformat() enrollment_end = obj.actual_enrollment_end_date - if obj.is_holdback and not obj.end_date and enrollment_end: - return enrollment_end.isoformat() return enrollment_end.isoformat() if enrollment_end else None def get_endDate(self, obj): - enrollment_end = obj.actual_enrollment_end_date - if obj.is_holdback and not obj.end_date and enrollment_end: - return (enrollment_end + datetime.timedelta(days=21)).isoformat() + holdback_end = self._holdback_enrollment_end(obj) + if holdback_end: + return ( + holdback_end + + datetime.timedelta(days=settings.HOLDBACK_OBSERVATION_DAYS) + ).isoformat() return obj.end_date.isoformat() if obj.end_date else None def get_proposedEnrollment(self, obj): - enrollment_end = obj.actual_enrollment_end_date - if obj.is_holdback and not obj.end_date and enrollment_end and obj.start_date: - return (enrollment_end - obj.start_date).days + holdback_end = self._holdback_enrollment_end(obj) + if holdback_end and obj.start_date: + return (holdback_end - obj.start_date).days return obj.proposed_enrollment def get_application(self, obj): diff --git a/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py b/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py index f66d40fb50..5bd766a87a 100644 --- a/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py +++ b/experimenter/experimenter/experiments/tests/api/v8/test_serializers.py @@ -419,24 +419,20 @@ def test_localized_localizations_json(self, l10n_json, expected): def test_holdback_serializer_overrides(self): today = datetime.date.today() start = today - datetime.timedelta(days=50) - enrollment_end = today - datetime.timedelta(days=21) + expected_enrollment_end = today - datetime.timedelta(days=21) experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, is_holdback=True, _start_date=start, - _enrollment_end_date=enrollment_end, ) serializer = NimbusExperimentSerializer(experiment) data = serializer.data - self.assertEqual(data["enrollmentEndDate"], enrollment_end.isoformat()) - self.assertEqual( - data["endDate"], - (enrollment_end + datetime.timedelta(days=21)).isoformat(), - ) + self.assertEqual(data["enrollmentEndDate"], expected_enrollment_end.isoformat()) + self.assertEqual(data["endDate"], today.isoformat()) self.assertEqual( data["proposedEnrollment"], - (enrollment_end - start).days, + (expected_enrollment_end - start).days, ) def test_non_holdback_proposed_enrollment_uses_model_value(self): diff --git a/experimenter/experimenter/jetstream/tasks.py b/experimenter/experimenter/jetstream/tasks.py index 31f83bc113..983837a160 100644 --- a/experimenter/experimenter/jetstream/tasks.py +++ b/experimenter/experimenter/jetstream/tasks.py @@ -198,9 +198,6 @@ def fetch_monitoring_data(): raise -HOLDBACK_OBSERVATION_DAYS = 21 - - @app.task @metrics.timer_decorator("update_holdback_enrollment_period") def update_holdback_enrollment_period(): @@ -208,7 +205,7 @@ def update_holdback_enrollment_period(): try: today = timezone.now().date() now = timezone.now() - enrollment_end = today - dt.timedelta(days=HOLDBACK_OBSERVATION_DAYS) + enrollment_end = today - dt.timedelta(days=settings.HOLDBACK_OBSERVATION_DAYS) experiments = NimbusExperiment.objects.filter( is_holdback=True, @@ -222,7 +219,6 @@ def update_holdback_enrollment_period(): continue NimbusExperiment.objects.filter(pk=experiment.pk).update( - _enrollment_end_date=enrollment_end, do_rerun=True, do_rerun_timestamp=now, ) diff --git a/experimenter/experimenter/jetstream/tests/test_tasks.py b/experimenter/experimenter/jetstream/tests/test_tasks.py index 282d222cf0..75046946f9 100644 --- a/experimenter/experimenter/jetstream/tests/test_tasks.py +++ b/experimenter/experimenter/jetstream/tests/test_tasks.py @@ -3952,7 +3952,7 @@ def test_result_is_cached(self): class TestUpdateHoldbackEnrollmentPeriod(TestCase): - def test_sets_enrollment_end_date_and_do_rerun(self): + def test_sets_do_rerun(self): today = datetime.date.today() start = today - datetime.timedelta(days=50) experiment = NimbusExperimentFactory.create_with_lifecycle( @@ -3965,10 +3965,9 @@ def test_sets_enrollment_end_date_and_do_rerun(self): tasks.update_holdback_enrollment_period() experiment.refresh_from_db() - expected_enrollment_end = today - datetime.timedelta(days=21) - self.assertEqual(experiment._enrollment_end_date, expected_enrollment_end) self.assertTrue(experiment.do_rerun) self.assertIsNotNone(experiment.do_rerun_timestamp) + self.assertIsNone(experiment._enrollment_end_date) def test_skips_experiment_started_within_observation_period(self): today = datetime.date.today() @@ -3982,7 +3981,6 @@ def test_skips_experiment_started_within_observation_period(self): tasks.update_holdback_enrollment_period() experiment.refresh_from_db() - self.assertIsNone(experiment._enrollment_end_date) self.assertFalse(experiment.do_rerun) def test_skips_ended_holdback(self): @@ -3999,7 +3997,6 @@ def test_skips_ended_holdback(self): tasks.update_holdback_enrollment_period() experiment.refresh_from_db() - self.assertIsNone(experiment._enrollment_end_date) self.assertFalse(experiment.do_rerun) def test_skips_non_holdback_experiments(self): diff --git a/experimenter/experimenter/settings.py b/experimenter/experimenter/settings.py index 54d073eabb..ba6f86a636 100644 --- a/experimenter/experimenter/settings.py +++ b/experimenter/experimenter/settings.py @@ -380,6 +380,8 @@ API_CACHE_WARMING_TTL = 60 * 60 * 24 SIZING_DATA_KEY = "population_sizing" +HOLDBACK_OBSERVATION_DAYS = 21 + # Celery CELERY_BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}" CELERY_BEAT_SCHEDULE = { From abbe729574b0ec356da22b8b30ff01c9fa8c38d3 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Tue, 7 Jul 2026 16:18:32 -0700 Subject: [PATCH 13/21] style(nimbus): format v8 serializer --- experimenter/experimenter/experiments/api/v8/serializers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/experimenter/experimenter/experiments/api/v8/serializers.py b/experimenter/experimenter/experiments/api/v8/serializers.py index 030f7bc77c..e51e7aa078 100644 --- a/experimenter/experimenter/experiments/api/v8/serializers.py +++ b/experimenter/experimenter/experiments/api/v8/serializers.py @@ -180,8 +180,7 @@ def get_endDate(self, obj): holdback_end = self._holdback_enrollment_end(obj) if holdback_end: return ( - holdback_end - + datetime.timedelta(days=settings.HOLDBACK_OBSERVATION_DAYS) + holdback_end + datetime.timedelta(days=settings.HOLDBACK_OBSERVATION_DAYS) ).isoformat() return obj.end_date.isoformat() if obj.end_date else None From 7b380029a7d8f3f5781ea5818e0de9a7a7d9ef90 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Tue, 7 Jul 2026 16:31:33 -0700 Subject: [PATCH 14/21] fix(nimbus): remove dead get_application method from v8 serializer --- experimenter/experimenter/experiments/api/v8/serializers.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/experimenter/experimenter/experiments/api/v8/serializers.py b/experimenter/experimenter/experiments/api/v8/serializers.py index e51e7aa078..982bce9f12 100644 --- a/experimenter/experimenter/experiments/api/v8/serializers.py +++ b/experimenter/experimenter/experiments/api/v8/serializers.py @@ -190,9 +190,6 @@ def get_proposedEnrollment(self, obj): return (holdback_end - obj.start_date).days return obj.proposed_enrollment - def get_application(self, obj): - return self.get_appId(obj) - def get_appName(self, obj): return obj.application_config.app_name From 6700eea59a6040aa71e563faf35d196295c2f0c2 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Wed, 8 Jul 2026 09:11:52 -0700 Subject: [PATCH 15/21] fix(nimbus): run holdback task weekly with 28-day minimum (7 enrollment + 21 observation) --- experimenter/experimenter/jetstream/tasks.py | 3 ++- experimenter/experimenter/jetstream/tests/test_tasks.py | 6 ++++-- experimenter/experimenter/settings.py | 3 ++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/experimenter/experimenter/jetstream/tasks.py b/experimenter/experimenter/jetstream/tasks.py index 983837a160..f709c0816b 100644 --- a/experimenter/experimenter/jetstream/tasks.py +++ b/experimenter/experimenter/jetstream/tasks.py @@ -215,7 +215,8 @@ def update_holdback_enrollment_period(): updated_count = 0 for experiment in experiments: - if enrollment_end <= experiment.start_date: + enrollment_days = (enrollment_end - experiment.start_date).days + if enrollment_days < settings.HOLDBACK_MINIMUM_ENROLLMENT_DAYS: continue NimbusExperiment.objects.filter(pk=experiment.pk).update( diff --git a/experimenter/experimenter/jetstream/tests/test_tasks.py b/experimenter/experimenter/jetstream/tests/test_tasks.py index 75046946f9..8c97f3b885 100644 --- a/experimenter/experimenter/jetstream/tests/test_tasks.py +++ b/experimenter/experimenter/jetstream/tests/test_tasks.py @@ -3969,12 +3969,14 @@ def test_sets_do_rerun(self): self.assertIsNotNone(experiment.do_rerun_timestamp) self.assertIsNone(experiment._enrollment_end_date) - def test_skips_experiment_started_within_observation_period(self): + def test_skips_experiment_below_minimum_enrollment(self): today = datetime.date.today() + # Started 25 days ago: enrollment_end = today - 21 = 4 days ago, + # enrollment_days = 4, which is < HOLDBACK_MINIMUM_ENROLLMENT_DAYS (7) experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, is_holdback=True, - _start_date=today - datetime.timedelta(days=10), + _start_date=today - datetime.timedelta(days=25), proposed_enrollment=14, proposed_duration=84, ) diff --git a/experimenter/experimenter/settings.py b/experimenter/experimenter/settings.py index ba6f86a636..17613bce94 100644 --- a/experimenter/experimenter/settings.py +++ b/experimenter/experimenter/settings.py @@ -381,6 +381,7 @@ SIZING_DATA_KEY = "population_sizing" HOLDBACK_OBSERVATION_DAYS = 21 +HOLDBACK_MINIMUM_ENROLLMENT_DAYS = 7 # Celery CELERY_BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}" @@ -425,7 +426,7 @@ }, "update_holdback_enrollment_period": { "task": "experimenter.jetstream.tasks.update_holdback_enrollment_period", - "schedule": crontab(minute=0, hour=7), + "schedule": crontab(minute=0, hour=7, day_of_week=1), }, } CELERY_TASK_ROUTES = { From db296aae04aaa1c19555ce2bf180f03d07d7e34e Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Wed, 8 Jul 2026 09:13:51 -0700 Subject: [PATCH 16/21] fix(nimbus): run holdback task daily but trigger weekly per experiment start date --- experimenter/experimenter/jetstream/tasks.py | 8 +++--- .../jetstream/tests/test_tasks.py | 25 +++++++++++++++---- experimenter/experimenter/settings.py | 2 +- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/experimenter/experimenter/jetstream/tasks.py b/experimenter/experimenter/jetstream/tasks.py index f709c0816b..889403fb1f 100644 --- a/experimenter/experimenter/jetstream/tasks.py +++ b/experimenter/experimenter/jetstream/tasks.py @@ -205,7 +205,6 @@ def update_holdback_enrollment_period(): try: today = timezone.now().date() now = timezone.now() - enrollment_end = today - dt.timedelta(days=settings.HOLDBACK_OBSERVATION_DAYS) experiments = NimbusExperiment.objects.filter( is_holdback=True, @@ -213,10 +212,13 @@ def update_holdback_enrollment_period(): _end_date=None, ).exclude(_start_date=None) + minimum_days = ( + settings.HOLDBACK_OBSERVATION_DAYS + settings.HOLDBACK_MINIMUM_ENROLLMENT_DAYS + ) updated_count = 0 for experiment in experiments: - enrollment_days = (enrollment_end - experiment.start_date).days - if enrollment_days < settings.HOLDBACK_MINIMUM_ENROLLMENT_DAYS: + days_since_start = (today - experiment.start_date).days + if days_since_start < minimum_days or days_since_start % 7 != 0: continue NimbusExperiment.objects.filter(pk=experiment.pk).update( diff --git a/experimenter/experimenter/jetstream/tests/test_tasks.py b/experimenter/experimenter/jetstream/tests/test_tasks.py index 8c97f3b885..800c4b1429 100644 --- a/experimenter/experimenter/jetstream/tests/test_tasks.py +++ b/experimenter/experimenter/jetstream/tests/test_tasks.py @@ -3954,7 +3954,8 @@ def test_result_is_cached(self): class TestUpdateHoldbackEnrollmentPeriod(TestCase): def test_sets_do_rerun(self): today = datetime.date.today() - start = today - datetime.timedelta(days=50) + # 28 days ago: 28 >= 28 minimum and 28 % 7 == 0, so should trigger + start = today - datetime.timedelta(days=28) experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, is_holdback=True, @@ -3969,14 +3970,28 @@ def test_sets_do_rerun(self): self.assertIsNotNone(experiment.do_rerun_timestamp) self.assertIsNone(experiment._enrollment_end_date) - def test_skips_experiment_below_minimum_enrollment(self): + def test_skips_experiment_below_minimum_days(self): today = datetime.date.today() - # Started 25 days ago: enrollment_end = today - 21 = 4 days ago, - # enrollment_days = 4, which is < HOLDBACK_MINIMUM_ENROLLMENT_DAYS (7) + # Started 20 days ago: 20 < 28 (minimum), so should be skipped experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, is_holdback=True, - _start_date=today - datetime.timedelta(days=25), + _start_date=today - datetime.timedelta(days=20), + proposed_enrollment=14, + proposed_duration=84, + ) + tasks.update_holdback_enrollment_period() + experiment.refresh_from_db() + + self.assertFalse(experiment.do_rerun) + + def test_skips_experiment_not_on_weekly_boundary(self): + today = datetime.date.today() + # Started 29 days ago: >= 28 but 29 % 7 != 0, so should be skipped + experiment = NimbusExperimentFactory.create_with_lifecycle( + NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, + is_holdback=True, + _start_date=today - datetime.timedelta(days=29), proposed_enrollment=14, proposed_duration=84, ) diff --git a/experimenter/experimenter/settings.py b/experimenter/experimenter/settings.py index 17613bce94..51be1a93f6 100644 --- a/experimenter/experimenter/settings.py +++ b/experimenter/experimenter/settings.py @@ -426,7 +426,7 @@ }, "update_holdback_enrollment_period": { "task": "experimenter.jetstream.tasks.update_holdback_enrollment_period", - "schedule": crontab(minute=0, hour=7, day_of_week=1), + "schedule": crontab(minute=0, hour=7), }, } CELERY_TASK_ROUTES = { From d35ce7ad90353199b7e9f01152827cd8c1b88c62 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Wed, 8 Jul 2026 09:36:20 -0700 Subject: [PATCH 17/21] test(nimbus): add test for holdback do_rerun at 35-day boundary --- .../experimenter/jetstream/tests/test_tasks.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/experimenter/experimenter/jetstream/tests/test_tasks.py b/experimenter/experimenter/jetstream/tests/test_tasks.py index 800c4b1429..44066ed177 100644 --- a/experimenter/experimenter/jetstream/tests/test_tasks.py +++ b/experimenter/experimenter/jetstream/tests/test_tasks.py @@ -3970,6 +3970,22 @@ def test_sets_do_rerun(self): self.assertIsNotNone(experiment.do_rerun_timestamp) self.assertIsNone(experiment._enrollment_end_date) + def test_sets_do_rerun_at_35_days(self): + today = datetime.date.today() + # 35 days ago: 35 >= 28 minimum and 35 % 7 == 0, so should trigger + experiment = NimbusExperimentFactory.create_with_lifecycle( + NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, + is_holdback=True, + _start_date=today - datetime.timedelta(days=35), + proposed_enrollment=14, + proposed_duration=84, + ) + tasks.update_holdback_enrollment_period() + experiment.refresh_from_db() + + self.assertTrue(experiment.do_rerun) + self.assertIsNotNone(experiment.do_rerun_timestamp) + def test_skips_experiment_below_minimum_days(self): today = datetime.date.today() # Started 20 days ago: 20 < 28 (minimum), so should be skipped From 38cafd8cba3c2c94883fa061871a8dcc9c8a6e75 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Wed, 8 Jul 2026 10:41:33 -0700 Subject: [PATCH 18/21] fix(nimbus): set do_rerun once then update only timestamp weekly, skip if enrollment stopped --- experimenter/experimenter/jetstream/tasks.py | 9 +++--- .../jetstream/tests/test_tasks.py | 28 ++++++++++++++----- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/experimenter/experimenter/jetstream/tasks.py b/experimenter/experimenter/jetstream/tasks.py index 889403fb1f..40fd51729d 100644 --- a/experimenter/experimenter/jetstream/tasks.py +++ b/experimenter/experimenter/jetstream/tasks.py @@ -210,6 +210,7 @@ def update_holdback_enrollment_period(): is_holdback=True, status=NimbusExperiment.Status.LIVE, _end_date=None, + _enrollment_end_date=None, ).exclude(_start_date=None) minimum_days = ( @@ -221,10 +222,10 @@ def update_holdback_enrollment_period(): if days_since_start < minimum_days or days_since_start % 7 != 0: continue - NimbusExperiment.objects.filter(pk=experiment.pk).update( - do_rerun=True, - do_rerun_timestamp=now, - ) + update_fields = {"do_rerun_timestamp": now} + if not experiment.do_rerun: + update_fields["do_rerun"] = True + NimbusExperiment.objects.filter(pk=experiment.pk).update(**update_fields) experiment.refresh_from_db() generate_nimbus_changelog( experiment, diff --git a/experimenter/experimenter/jetstream/tests/test_tasks.py b/experimenter/experimenter/jetstream/tests/test_tasks.py index 44066ed177..c959546369 100644 --- a/experimenter/experimenter/jetstream/tests/test_tasks.py +++ b/experimenter/experimenter/jetstream/tests/test_tasks.py @@ -3952,14 +3952,13 @@ def test_result_is_cached(self): class TestUpdateHoldbackEnrollmentPeriod(TestCase): - def test_sets_do_rerun(self): + def test_sets_do_rerun_on_first_trigger(self): today = datetime.date.today() - # 28 days ago: 28 >= 28 minimum and 28 % 7 == 0, so should trigger - start = today - datetime.timedelta(days=28) + # 28 days ago: first trigger — sets both do_rerun and do_rerun_timestamp experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, is_holdback=True, - _start_date=start, + _start_date=today - datetime.timedelta(days=28), proposed_enrollment=14, proposed_duration=84, ) @@ -3968,17 +3967,17 @@ def test_sets_do_rerun(self): self.assertTrue(experiment.do_rerun) self.assertIsNotNone(experiment.do_rerun_timestamp) - self.assertIsNone(experiment._enrollment_end_date) - def test_sets_do_rerun_at_35_days(self): + def test_updates_only_timestamp_on_subsequent_triggers(self): today = datetime.date.today() - # 35 days ago: 35 >= 28 minimum and 35 % 7 == 0, so should trigger + # 35 days: do_rerun already True — only do_rerun_timestamp should update experiment = NimbusExperimentFactory.create_with_lifecycle( NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, is_holdback=True, _start_date=today - datetime.timedelta(days=35), proposed_enrollment=14, proposed_duration=84, + do_rerun=True, ) tasks.update_holdback_enrollment_period() experiment.refresh_from_db() @@ -3986,6 +3985,21 @@ def test_sets_do_rerun_at_35_days(self): self.assertTrue(experiment.do_rerun) self.assertIsNotNone(experiment.do_rerun_timestamp) + def test_skips_holdback_with_enrollment_stopped(self): + today = datetime.date.today() + experiment = NimbusExperimentFactory.create_with_lifecycle( + NimbusExperimentFactory.Lifecycles.LIVE_ENROLLING, + is_holdback=True, + _start_date=today - datetime.timedelta(days=35), + _enrollment_end_date=today - datetime.timedelta(days=7), + proposed_enrollment=14, + proposed_duration=84, + ) + tasks.update_holdback_enrollment_period() + experiment.refresh_from_db() + + self.assertFalse(experiment.do_rerun) + def test_skips_experiment_below_minimum_days(self): today = datetime.date.today() # Started 20 days ago: 20 < 28 (minimum), so should be skipped From 27198fc07efa8719bec4efed55606c4446b78f76 Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Wed, 8 Jul 2026 11:01:36 -0700 Subject: [PATCH 19/21] fix(nimbus): move weekly interval to HOLDBACK_RERUN_INTERVAL_DAYS constant --- experimenter/experimenter/jetstream/tasks.py | 2 +- experimenter/experimenter/settings.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/experimenter/experimenter/jetstream/tasks.py b/experimenter/experimenter/jetstream/tasks.py index 40fd51729d..dce2f91278 100644 --- a/experimenter/experimenter/jetstream/tasks.py +++ b/experimenter/experimenter/jetstream/tasks.py @@ -219,7 +219,7 @@ def update_holdback_enrollment_period(): updated_count = 0 for experiment in experiments: days_since_start = (today - experiment.start_date).days - if days_since_start < minimum_days or days_since_start % 7 != 0: + if days_since_start < minimum_days or days_since_start % settings.HOLDBACK_RERUN_INTERVAL_DAYS != 0: continue update_fields = {"do_rerun_timestamp": now} diff --git a/experimenter/experimenter/settings.py b/experimenter/experimenter/settings.py index 51be1a93f6..4dd43b401e 100644 --- a/experimenter/experimenter/settings.py +++ b/experimenter/experimenter/settings.py @@ -382,6 +382,7 @@ HOLDBACK_OBSERVATION_DAYS = 21 HOLDBACK_MINIMUM_ENROLLMENT_DAYS = 7 +HOLDBACK_RERUN_INTERVAL_DAYS = 7 # Celery CELERY_BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}" From 60abfd9a8a9e2c3639ab5c78fcd6b56e0b2e1abf Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Wed, 8 Jul 2026 11:03:18 -0700 Subject: [PATCH 20/21] style(nimbus): format tasks.py --- experimenter/experimenter/jetstream/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/experimenter/experimenter/jetstream/tasks.py b/experimenter/experimenter/jetstream/tasks.py index dce2f91278..f64c56d31a 100644 --- a/experimenter/experimenter/jetstream/tasks.py +++ b/experimenter/experimenter/jetstream/tasks.py @@ -219,7 +219,10 @@ def update_holdback_enrollment_period(): updated_count = 0 for experiment in experiments: days_since_start = (today - experiment.start_date).days - if days_since_start < minimum_days or days_since_start % settings.HOLDBACK_RERUN_INTERVAL_DAYS != 0: + if ( + days_since_start < minimum_days + or days_since_start % settings.HOLDBACK_RERUN_INTERVAL_DAYS != 0 + ): continue update_fields = {"do_rerun_timestamp": now} From d1a933c865c522473cf42b5b7e0e2d71df25681f Mon Sep 17 00:00:00 2001 From: Yashika Khurana Date: Wed, 8 Jul 2026 15:12:06 -0700 Subject: [PATCH 21/21] fix(nimbus): run holdback task at 3am UTC, simplify save, add debug logging for skips --- experimenter/experimenter/jetstream/tasks.py | 15 +++++++++++---- experimenter/experimenter/settings.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/experimenter/experimenter/jetstream/tasks.py b/experimenter/experimenter/jetstream/tasks.py index f64c56d31a..c69dedb777 100644 --- a/experimenter/experimenter/jetstream/tasks.py +++ b/experimenter/experimenter/jetstream/tasks.py @@ -223,13 +223,20 @@ def update_holdback_enrollment_period(): days_since_start < minimum_days or days_since_start % settings.HOLDBACK_RERUN_INTERVAL_DAYS != 0 ): + logger.debug( + f"Skipping holdback {experiment.slug}: " + f"days_since_start={days_since_start}, " + f"minimum={minimum_days}, " + f"interval={settings.HOLDBACK_RERUN_INTERVAL_DAYS}" + ) continue - update_fields = {"do_rerun_timestamp": now} + save_fields = ["do_rerun_timestamp"] if not experiment.do_rerun: - update_fields["do_rerun"] = True - NimbusExperiment.objects.filter(pk=experiment.pk).update(**update_fields) - experiment.refresh_from_db() + experiment.do_rerun = True + save_fields.append("do_rerun") + experiment.do_rerun_timestamp = now + experiment.save(update_fields=save_fields) generate_nimbus_changelog( experiment, get_kinto_user(), diff --git a/experimenter/experimenter/settings.py b/experimenter/experimenter/settings.py index 4dd43b401e..c2ed5991a1 100644 --- a/experimenter/experimenter/settings.py +++ b/experimenter/experimenter/settings.py @@ -427,7 +427,7 @@ }, "update_holdback_enrollment_period": { "task": "experimenter.jetstream.tasks.update_holdback_enrollment_period", - "schedule": crontab(minute=0, hour=7), + "schedule": crontab(minute=0, hour=3), }, } CELERY_TASK_ROUTES = {