diff --git a/benchmarking/locust/common/ateapi_pb2.py b/benchmarking/locust/common/ateapi_pb2.py index 576745cc7d..09b502c6cf 100644 --- a/benchmarking/locust/common/ateapi_pb2.py +++ b/benchmarking/locust/common/ateapi_pb2.py @@ -40,7 +40,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x61teapi.proto\x12\x06\x61teapi\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"]\n\x10\x45xternalSnapshot\x12\x14\n\x0csnapshot_uri\x18\x01 \x01(\t\x12\x33\n\rcontent_scope\x18\x02 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\"\x86\x01\n\x11LocalSnapshotInfo\x12\x15\n\rsnapshot_name\x18\x01 \x01(\t\x12%\n\x1dnode_vms_with_local_snapshots\x18\x02 \x03(\t\x12\x33\n\rcontent_scope\x18\x03 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\"w\n\x08Selector\x12\x37\n\x0cmatch_labels\x18\x01 \x03(\x0b\x32!.ateapi.Selector.MatchLabelsEntry\x1a\x32\n\x10MatchLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb2\x01\n\x10ResourceMetadata\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0b\n\x03uid\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\x03\x12/\n\x0b\x63reate_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\x0bupdate_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xdc\x02\n\x0e\x45xternalVolume\x12\x13\n\x0bvolume_name\x18\x01 \x01(\t\x12\x19\n\x11storage_volume_id\x18\x02 \x01(\t\x12\x13\n\x0bvolume_type\x18\x03 \x01(\t\x12-\n\x06status\x18\x04 \x01(\x0e\x32\x1d.ateapi.ExternalVolume.Status\x12\x41\n\x0evolume_context\x18\x05 \x03(\x0b\x32).ateapi.ExternalVolume.VolumeContextEntry\x1a\x34\n\x12VolumeContextEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x12\n\x0eSTATUS_PENDING\x10\x01\x12\x12\n\x0eSTATUS_CREATED\x10\x02\x12\x13\n\x0fSTATUS_DELETING\x10\x03\"\xd5\x01\n\x05\x41\x63tor\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0e\x61\x63tor_template\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12)\n\x0fworker_selector\x18\x05 \x01(\x0b\x32\x10.ateapi.Selector\x12%\n\nsource_tag\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12#\n\x06status\x18\x07 \x01(\x0b\x32\x13.ateapi.ActorStatus\"]\n\x0c\x45gressPolicy\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12!\n\x05rules\x18\x02 \x03(\x0b\x32\x12.ateapi.EgressRule\"\x82\x01\n\nEgressRule\x12\'\n\thostnames\x18\x01 \x01(\x0b\x32\x14.ateapi.HostnameRule\x12&\n\tip_blocks\x18\x02 \x01(\x0b\x32\x13.ateapi.IPBlockRule\x12#\n\x03\x61ll\x18\x03 \x01(\x0b\x32\x16.google.protobuf.Empty\"L\n\x0cHostnameRule\x12\x10\n\x08patterns\x18\x01 \x03(\t\x12*\n\x07\x65\x66\x66\x65\x63ts\x18\x02 \x01(\x0b\x32\x19.ateapi.EgressRuleEffects\"\x1c\n\x0bIPBlockRule\x12\r\n\x05\x63idrs\x18\x01 \x03(\t\"U\n\x11\x45gressRuleEffects\x12@\n\x15inject_static_headers\x18\x01 \x03(\x0b\x32!.ateapi.CredentialHeaderInjection\"S\n\x19\x43redentialHeaderInjection\x12\x0e\n\x06header\x18\x01 \x01(\t\x12\x0e\n\x06prefix\x18\x02 \x01(\t\x12\x16\n\x0e\x63redential_uri\x18\x03 \x01(\t\"\xf1\x02\n\x0b\x41\x63torStatus\x12!\n\x05state\x18\x01 \x01(\x0e\x32\x12.ateapi.ActorState\x12\x33\n\x11worker_assignment\x18\x02 \x01(\x0b\x32\x18.ateapi.WorkerAssignment\x12!\n\x19in_progress_snapshot_name\x18\x03 \x01(\t\x12\x33\n\x11\x65xternal_snapshot\x18\x04 \x01(\x0b\x32\x18.ateapi.ExternalSnapshot\x12\x36\n\x13local_snapshot_info\x18\x05 \x01(\x0b\x32\x19.ateapi.LocalSnapshotInfo\x12-\n\ractor_volumes\x18\x07 \x03(\x0b\x32\x16.ateapi.ExternalVolume\x12\'\n\x1fin_progress_local_snapshot_name\x18\x08 \x01(\t\x12\"\n\x1a\x63urrent_actor_template_uid\x18\x0b \x01(\t\"\xa7\x01\n\x10WorkerAssignment\x12!\n\x06worker\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x18\n\x10worker_namespace\x18\x01 \x01(\t\x12\x13\n\x0bworker_pool\x18\x02 \x01(\t\x12\x12\n\nworker_pod\x18\x03 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x04 \x01(\t\x12\x15\n\rworker_pod_ip\x18\x05 \x01(\t\"\x8f\x01\n\tTagStatus\x12*\n\x08snapshot\x18\x01 \x01(\x0b\x32\x18.ateapi.ExternalSnapshot\x12\x1a\n\x12\x61\x63tor_template_uid\x18\x02 \x01(\t\x12 \n\x18in_progress_snapshot_uri\x18\x03 \x01(\t\x12\x18\n\x10source_actor_uid\x18\x04 \x01(\t\"\x9e\x01\n\x03Tag\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12!\n\x06status\x18\x02 \x01(\x0b\x32\x11.ateapi.TagStatus\x12\x1f\n\x05scope\x18\x03 \x01(\x0e\x32\x10.ateapi.TagScope\x12\'\n\x0csource_actor\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\"6\n\x08\x41tespace\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\"+\n\tObjectRef\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"\xe3\x02\n\rActorTemplate\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0fworker_selector\x18\x02 \x01(\x0b\x32\x10.ateapi.Selector\x12%\n\ncontainers\x18\x03 \x03(\x0b\x32\x11.ateapi.Container\x12\x1f\n\x07volumes\x18\x04 \x03(\x0b\x32\x0e.ateapi.Volume\x12\x31\n\x10snapshots_config\x18\x05 \x01(\x0b\x32\x17.ateapi.SnapshotsConfig\x12-\n\x0esandbox_config\x18\x06 \x01(\x0b\x32\x15.ateapi.SandboxConfig\x12$\n\tresources\x18\x07 \x01(\x0b\x32\x11.ateapi.Resources\x12+\n\x06status\x18\x08 \x01(\x0b\x32\x1b.ateapi.ActorTemplateStatus\"+\n\tResources\x12\x1e\n\x06limits\x18\x01 \x03(\x0b\x32\x0e.ateapi.Limits\"(\n\x06Limits\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08quantity\x18\x02 \x01(\t\"\x9d\x01\n\x14GoldenSnapshotStatus\x12\x31\n\x0fgolden_snapshot\x18\x01 \x01(\x0b\x32\x18.ateapi.ExternalSnapshot\x12;\n\x17take_golden_snapshot_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x15\n\rerror_message\x18\x03 \x01(\t\"S\n\x13\x41\x63torTemplateStatus\x12<\n\x16golden_snapshot_status\x18\x01 \x01(\x0b\x32\x1c.ateapi.GoldenSnapshotStatus\"Q\n\rSandboxConfig\x12+\n\rsandbox_class\x18\x01 \x01(\x0e\x32\x14.ateapi.SandboxClass\x12\x13\n\x0b\x63onfig_name\x18\x02 \x01(\t\"\xb7\x01\n\x0fSnapshotsConfig\x12.\n\x08on_pause\x18\x01 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12/\n\ton_commit\x18\x02 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12)\n\ton_resume\x18\x03 \x01(\x0b\x32\x16.ateapi.OnResumeConfig\x12\x18\n\x10storage_location\x18\x04 \x01(\t\"9\n\x0eOnResumeConfig\x12\'\n\tfrom_data\x18\x01 \x01(\x0e\x32\x14.ateapi.ResumeSource\"\x92\x02\n\tContainer\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05image\x18\x02 \x01(\t\x12\x0f\n\x07\x63ommand\x18\x03 \x03(\t\x12\x0c\n\x04\x61rgs\x18\x04 \x03(\t\x12\x1b\n\x03\x65nv\x18\x05 \x03(\x0b\x32\x0e.ateapi.EnvVar\x12\'\n\x06readyz\x18\x06 \x01(\x0b\x32\x17.ateapi.ContainerReadyz\x12*\n\rvolume_mounts\x18\x07 \x03(\x0b\x32\x13.ateapi.VolumeMount\x12\x31\n\x10security_context\x18\x08 \x01(\x0b\x32\x17.ateapi.SecurityContext\x12$\n\tresources\x18\t \x01(\x0b\x32\x11.ateapi.Resources\"=\n\x0fSecurityContext\x12*\n\x0c\x63\x61pabilities\x18\x01 \x01(\x0b\x32\x14.ateapi.Capabilities\")\n\x0c\x43\x61pabilities\x12\x0b\n\x03\x61\x64\x64\x18\x01 \x03(\t\x12\x0c\n\x04\x64rop\x18\x02 \x03(\t\"%\n\x06\x45nvVar\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"S\n\x0f\x43ontainerReadyz\x12\'\n\x08http_get\x18\x01 \x01(\x0b\x32\x15.ateapi.HTTPGetAction\x12\x17\n\x0ftimeout_seconds\x18\x02 \x01(\x05\"+\n\rHTTPGetAction\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xec\x01\n\x06Volume\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x33\n\x0b\x64urable_dir\x18\x02 \x01(\x0b\x32\x1e.ateapi.DurableDirVolumeSource\x12@\n\x18\x65xternal_volume_template\x18\x03 \x01(\x0b\x32\x1e.ateapi.ExternalVolumeTemplate\x12\x33\n\x0bsystem_info\x18\x05 \x01(\x0b\x32\x1e.ateapi.SystemInfoVolumeSource\x12(\n\x05image\x18\x06 \x01(\x0b\x32\x19.ateapi.ImageVolumeSource\"&\n\x11ImageVolumeSource\x12\x11\n\treference\x18\x01 \x01(\t\"\x18\n\x16\x44urableDirVolumeSource\"F\n\x16\x45xternalVolumeTemplate\x12\x10\n\x08\x63\x61pacity\x18\x01 \x01(\t\x12\x1a\n\x12storage_class_name\x18\x02 \x01(\t\"L\n\x16SystemInfoVolumeSource\x12\x32\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x1c.ateapi.SystemInfoDataSource\"\x84\x01\n\x14SystemInfoDataSource\x12\x37\n\x0e\x61\x63tor_metadata\x18\x01 \x01(\x0b\x32\x1f.ateapi.ActorMetadataDataSource\x12\x33\n\x0ctrust_bundle\x18\x02 \x01(\x0b\x32\x1d.ateapi.TrustBundleDataSource\"C\n\x17\x41\x63torMetadataDataSource\x12(\n\x05items\x18\x01 \x03(\x0b\x32\x19.ateapi.ActorMetadataItem\"L\n\x11\x41\x63torMetadataItem\x12)\n\x05\x66ield\x18\x01 \x01(\x0e\x32\x1a.ateapi.ActorMetadataField\x12\x0c\n\x04path\x18\x02 \x01(\t\"3\n\x15TrustBundleDataSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\"/\n\x0bVolumeMount\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nmount_path\x18\x02 \x01(\t\";\n\x15\x43reateAtespaceRequest\x12\"\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x10.ateapi.Atespace\"9\n\x12GetAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"=\n\x14ListAtespacesRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"U\n\x15ListAtespacesResponse\x12#\n\tatespaces\x18\x01 \x03(\x0b\x32\x10.ateapi.Atespace\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"<\n\x15\x44\x65leteAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"K\n\x1a\x43reateActorTemplateRequest\x12-\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x15.ateapi.ActorTemplate\"D\n\x17GetActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"T\n\x19ListActorTemplatesRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"e\n\x1aListActorTemplatesResponse\x12.\n\x0f\x61\x63tor_templates\x18\x01 \x03(\x0b\x32\x15.ateapi.ActorTemplate\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"G\n\x1a\x44\x65leteActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"3\n\x0fGetActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12\x43reateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"2\n\x12UpdateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"7\n\x13SuspendActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"4\n\x14SuspendActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"5\n\x11PauseActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12PauseActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"D\n\x12ResumeActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x0c\n\x04\x62oot\x18\x02 \x01(\x08\"D\n\x13ResumeActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\x12\x0f\n\x07resumed\x18\x02 \x01(\x08\"I\n\x12\x44\x65leteActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tany_state\x18\x02 \x01(\x08\"?\n\x1bGetActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"o\n\x1e\x43reateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"o\n\x1eUpdateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"B\n\x1e\x44\x65leteActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"/\n\rGetTagRequest\x12\x1e\n\x03tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"J\n\x0fListTagsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"F\n\x10ListTagsResponse\x12\x19\n\x04tags\x18\x01 \x03(\x0b\x32\x0b.ateapi.Tag\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\",\n\x10\x43reateTagRequest\x12\x18\n\x03tag\x18\x01 \x01(\x0b\x32\x0b.ateapi.Tag\",\n\x10UpdateTagRequest\x12\x18\n\x03tag\x18\x01 \x01(\x0b\x32\x0b.ateapi.Tag\"2\n\x10\x44\x65leteTagRequest\x12\x1e\n\x03tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"-\n\rDeleteOptions\x12\x0f\n\x07version\x18\x01 \x01(\x03\x12\x0b\n\x03uid\x18\x02 \x01(\t\"m\n!ListWorkerActorAssignmentsRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"q\n\"ListWorkerActorAssignmentsResponse\x12\x32\n\x11\x61\x63tor_assignments\x18\x01 \x03(\x0b\x32\x17.ateapi.ActorAssignment\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\";\n\x12ListWorkersRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"O\n\x13ListWorkersResponse\x12\x1f\n\x07workers\x18\x01 \x03(\x0b\x32\x0e.ateapi.Worker\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"5\n\x10GetWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"5\n\x13\x43reateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"5\n\x13UpdateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"`\n\x13\x44\x65leteWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12&\n\x07options\x18\x02 \x01(\x0b\x32\x15.ateapi.DeleteOptions\"7\n\x12\x44rainWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"L\n\x11ListActorsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"L\n\x12ListActorsResponse\x12\x1d\n\x06\x61\x63tors\x18\x01 \x03(\x0b\x32\r.ateapi.Actor\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"\xc6\x02\n\x06Worker\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12\x18\n\x10worker_namespace\x18\x02 \x01(\t\x12\x13\n\x0bworker_pool\x18\x03 \x01(\t\x12\x12\n\nworker_pod\x18\x04 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x05 \x01(\t\x12\x11\n\tnode_name\x18\x06 \x01(\t\x12\n\n\x02ip\x18\x07 \x01(\t\x12\x15\n\rsandbox_class\x18\x08 \x01(\t\x12*\n\x06labels\x18\t \x03(\x0b\x32\x1a.ateapi.Worker.LabelsEntry\x12$\n\x06status\x18\x0b \x01(\x0b\x32\x14.ateapi.WorkerStatus\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x89\x01\n\x0cWorkerStatus\x12\"\n\x05state\x18\x01 \x01(\x0e\x32\x13.ateapi.WorkerState\x12)\n\x08\x63\x61pacity\x18\x02 \x01(\x0b\x32\x17.ateapi.WorkerResources\x12*\n\tallocated\x18\x03 \x01(\x0b\x32\x17.ateapi.WorkerResources\"G\n\x0fWorkerResources\x12$\n\tresources\x18\x01 \x01(\x0b\x32\x11.ateapi.Resources\x12\x0e\n\x06\x61\x63tors\x18\x02 \x01(\x05\"\xc7\x01\n\x0f\x41\x63torAssignment\x12*\n\x08metadata\x18\x06 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12 \n\x05\x61\x63tor\x18\x02 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tactor_uid\x18\x03 \x01(\t\x12-\n\x12\x61\x63tor_template_ref\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12$\n\tresources\x18\x05 \x01(\x0b\x32\x11.ateapi.Resources\"h\n\x18SetWorkerCapacityRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12)\n\x08\x63\x61pacity\x18\x02 \x01(\x0b\x32\x17.ateapi.WorkerResources\";\n\x19SetWorkerCapacityResponse\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"[\n\x0eMintJWTRequest\x12\x10\n\x08\x61udience\x18\x01 \x03(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12\x12\n\nactor_name\x18\x03 \x01(\t\x12\x11\n\tactor_uid\x18\x04 \x01(\t\"$\n\x0fMintJWTResponse\x12\x11\n\tactor_jwt\x18\x01 \x01(\t\"\xa7\x01\n\x0fMintCertRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12#\n\x1b\x63\x65rtificate_signing_request\x18\x02 \x01(\x0c\x12\x1a\n\x12\x65xpected_actor_uid\x18\x03 \x01(\t\x12\x30\n\x07purpose\x18\x04 \x01(\x0e\x32\x1f.ateapi.ActorCertificatePurpose\".\n\x10MintCertResponse\x12\x1a\n\x12\x61\x63tor_certificates\x18\x01 \x03(\x0c*\x80\x01\n\x14SnapshotContentScope\x12&\n\"SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_FULL\x10\x01\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_DATA\x10\x02*V\n\x08TagScope\x12\x19\n\x15TAG_SCOPE_UNSPECIFIED\x10\x00\x12\x16\n\x12TAG_SCOPE_ATESPACE\x10\x01\x12\x17\n\x13TAG_SCOPE_PUBLISHED\x10\x02*\xf7\x01\n\nActorState\x12\x1b\n\x17\x41\x43TOR_STATE_UNSPECIFIED\x10\x00\x12\x18\n\x14\x41\x43TOR_STATE_RESUMING\x10\x01\x12\x17\n\x13\x41\x43TOR_STATE_RUNNING\x10\x02\x12\x1a\n\x16\x41\x43TOR_STATE_SUSPENDING\x10\x03\x12\x19\n\x15\x41\x43TOR_STATE_SUSPENDED\x10\x04\x12\x17\n\x13\x41\x43TOR_STATE_PAUSING\x10\x05\x12\x16\n\x12\x41\x43TOR_STATE_PAUSED\x10\x06\x12\x17\n\x13\x41\x43TOR_STATE_CRASHED\x10\x07\x12\x18\n\x14\x41\x43TOR_STATE_DELETING\x10\x08*b\n\x0cSandboxClass\x12\x1d\n\x19SANDBOX_CLASS_UNSPECIFIED\x10\x00\x12\x18\n\x14SANDBOX_CLASS_GVISOR\x10\x01\x12\x19\n\x15SANDBOX_CLASS_MICROVM\x10\x02*d\n\x0cResumeSource\x12\x1d\n\x19RESUME_SOURCE_UNSPECIFIED\x10\x00\x12\x1b\n\x17RESUME_SOURCE_COLD_BOOT\x10\x01\x12\x18\n\x14RESUME_SOURCE_GOLDEN\x10\x02*\x9a\x01\n\x12\x41\x63torMetadataField\x12$\n ACTOR_METADATA_FIELD_UNSPECIFIED\x10\x00\x12\x1d\n\x19\x41\x43TOR_METADATA_FIELD_NAME\x10\x01\x12!\n\x1d\x41\x43TOR_METADATA_FIELD_ATESPACE\x10\x02\x12\x1c\n\x18\x41\x43TOR_METADATA_FIELD_UID\x10\x03*_\n\x0bWorkerState\x12\x1c\n\x18WORKER_STATE_UNSPECIFIED\x10\x00\x12\x17\n\x13WORKER_STATE_ACTIVE\x10\x01\x12\x19\n\x15WORKER_STATE_DRAINING\x10\x02*k\n\x17\x41\x63torCertificatePurpose\x12)\n%ACTOR_CERTIFICATE_PURPOSE_UNSPECIFIED\x10\x00\x12%\n!ACTOR_CERTIFICATE_PURPOSE_ATUNNEL\x10\x01\x32\xf3\x11\n\x07\x43ontrol\x12\x34\n\x08GetActor\x12\x17.ateapi.GetActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0b\x43reateActor\x12\x1a.ateapi.CreateActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0bUpdateActor\x12\x1a.ateapi.UpdateActorRequest\x1a\r.ateapi.Actor\"\x00\x12K\n\x0cSuspendActor\x12\x1b.ateapi.SuspendActorRequest\x1a\x1c.ateapi.SuspendActorResponse\"\x00\x12\x45\n\nPauseActor\x12\x19.ateapi.PauseActorRequest\x1a\x1a.ateapi.PauseActorResponse\"\x00\x12H\n\x0bResumeActor\x12\x1a.ateapi.ResumeActorRequest\x1a\x1b.ateapi.ResumeActorResponse\"\x00\x12:\n\x0b\x44\x65leteActor\x12\x1a.ateapi.DeleteActorRequest\x1a\r.ateapi.Actor\"\x00\x12S\n\x14GetActorEgressPolicy\x12#.ateapi.GetActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x43reateActorEgressPolicy\x12&.ateapi.CreateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17UpdateActorEgressPolicy\x12&.ateapi.UpdateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x44\x65leteActorEgressPolicy\x12&.ateapi.DeleteActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12\x34\n\tCreateTag\x12\x18.ateapi.CreateTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12.\n\x06GetTag\x12\x15.ateapi.GetTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12?\n\x08ListTags\x12\x17.ateapi.ListTagsRequest\x1a\x18.ateapi.ListTagsResponse\"\x00\x12\x34\n\tUpdateTag\x12\x18.ateapi.UpdateTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12\x34\n\tDeleteTag\x12\x18.ateapi.DeleteTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12H\n\x0bListWorkers\x12\x1a.ateapi.ListWorkersRequest\x1a\x1b.ateapi.ListWorkersResponse\"\x00\x12\x37\n\tGetWorker\x12\x18.ateapi.GetWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x43reateWorker\x12\x1b.ateapi.CreateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0cUpdateWorker\x12\x1b.ateapi.UpdateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x44\x65leteWorker\x12\x1b.ateapi.DeleteWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12;\n\x0b\x44rainWorker\x12\x1a.ateapi.DrainWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12u\n\x1aListWorkerActorAssignments\x12).ateapi.ListWorkerActorAssignmentsRequest\x1a*.ateapi.ListWorkerActorAssignmentsResponse\"\x00\x12\x45\n\nListActors\x12\x19.ateapi.ListActorsRequest\x1a\x1a.ateapi.ListActorsResponse\"\x00\x12\x43\n\x0e\x43reateAtespace\x12\x1d.ateapi.CreateAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12=\n\x0bGetAtespace\x12\x1a.ateapi.GetAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12N\n\rListAtespaces\x12\x1c.ateapi.ListAtespacesRequest\x1a\x1d.ateapi.ListAtespacesResponse\"\x00\x12\x43\n\x0e\x44\x65leteAtespace\x12\x1d.ateapi.DeleteAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12R\n\x13\x43reateActorTemplate\x12\".ateapi.CreateActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12L\n\x10GetActorTemplate\x12\x1f.ateapi.GetActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12]\n\x12ListActorTemplates\x12!.ateapi.ListActorTemplatesRequest\x1a\".ateapi.ListActorTemplatesResponse\"\x00\x12R\n\x13\x44\x65leteActorTemplate\x12\".ateapi.DeleteActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x32\x8a\x01\n\rActorIdentity\x12:\n\x07MintJWT\x12\x16.ateapi.MintJWTRequest\x1a\x17.ateapi.MintJWTResponse\x12=\n\x08MintCert\x12\x17.ateapi.MintCertRequest\x1a\x18.ateapi.MintCertResponse2i\n\rWorkerService\x12X\n\x11SetWorkerCapacity\x12 .ateapi.SetWorkerCapacityRequest\x1a!.ateapi.SetWorkerCapacityResponseB9Z7github.com/agent-substrate/substrate/pkg/proto/ateapipbb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x61teapi.proto\x12\x06\x61teapi\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"]\n\x10\x45xternalSnapshot\x12\x14\n\x0csnapshot_uri\x18\x01 \x01(\t\x12\x33\n\rcontent_scope\x18\x02 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\"\x86\x01\n\x11LocalSnapshotInfo\x12\x15\n\rsnapshot_name\x18\x01 \x01(\t\x12%\n\x1dnode_vms_with_local_snapshots\x18\x02 \x03(\t\x12\x33\n\rcontent_scope\x18\x03 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\"w\n\x08Selector\x12\x37\n\x0cmatch_labels\x18\x01 \x03(\x0b\x32!.ateapi.Selector.MatchLabelsEntry\x1a\x32\n\x10MatchLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xb2\x01\n\x10ResourceMetadata\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0b\n\x03uid\x18\x03 \x01(\t\x12\x0f\n\x07version\x18\x04 \x01(\x03\x12/\n\x0b\x63reate_time\x18\x05 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12/\n\x0bupdate_time\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xdc\x02\n\x0e\x45xternalVolume\x12\x13\n\x0bvolume_name\x18\x01 \x01(\t\x12\x19\n\x11storage_volume_id\x18\x02 \x01(\t\x12\x13\n\x0bvolume_type\x18\x03 \x01(\t\x12-\n\x06status\x18\x04 \x01(\x0e\x32\x1d.ateapi.ExternalVolume.Status\x12\x41\n\x0evolume_context\x18\x05 \x03(\x0b\x32).ateapi.ExternalVolume.VolumeContextEntry\x1a\x34\n\x12VolumeContextEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x06Status\x12\x16\n\x12STATUS_UNSPECIFIED\x10\x00\x12\x12\n\x0eSTATUS_PENDING\x10\x01\x12\x12\n\x0eSTATUS_CREATED\x10\x02\x12\x13\n\x0fSTATUS_DELETING\x10\x03\"\xd5\x01\n\x05\x41\x63tor\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0e\x61\x63tor_template\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12)\n\x0fworker_selector\x18\x05 \x01(\x0b\x32\x10.ateapi.Selector\x12%\n\nsource_tag\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12#\n\x06status\x18\x07 \x01(\x0b\x32\x13.ateapi.ActorStatus\"]\n\x0c\x45gressPolicy\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12!\n\x05rules\x18\x02 \x03(\x0b\x32\x12.ateapi.EgressRule\"\x82\x01\n\nEgressRule\x12\'\n\thostnames\x18\x01 \x01(\x0b\x32\x14.ateapi.HostnameRule\x12&\n\tip_blocks\x18\x02 \x01(\x0b\x32\x13.ateapi.IPBlockRule\x12#\n\x03\x61ll\x18\x03 \x01(\x0b\x32\x16.google.protobuf.Empty\"L\n\x0cHostnameRule\x12\x10\n\x08patterns\x18\x01 \x03(\t\x12*\n\x07\x65\x66\x66\x65\x63ts\x18\x02 \x01(\x0b\x32\x19.ateapi.EgressRuleEffects\"\x1c\n\x0bIPBlockRule\x12\r\n\x05\x63idrs\x18\x01 \x03(\t\"U\n\x11\x45gressRuleEffects\x12@\n\x15inject_static_headers\x18\x01 \x03(\x0b\x32!.ateapi.CredentialHeaderInjection\"S\n\x19\x43redentialHeaderInjection\x12\x0e\n\x06header\x18\x01 \x01(\t\x12\x0e\n\x06prefix\x18\x02 \x01(\t\x12\x16\n\x0e\x63redential_uri\x18\x03 \x01(\t\"\xf1\x02\n\x0b\x41\x63torStatus\x12!\n\x05state\x18\x01 \x01(\x0e\x32\x12.ateapi.ActorState\x12\x33\n\x11worker_assignment\x18\x02 \x01(\x0b\x32\x18.ateapi.WorkerAssignment\x12!\n\x19in_progress_snapshot_name\x18\x03 \x01(\t\x12\x33\n\x11\x65xternal_snapshot\x18\x04 \x01(\x0b\x32\x18.ateapi.ExternalSnapshot\x12\x36\n\x13local_snapshot_info\x18\x05 \x01(\x0b\x32\x19.ateapi.LocalSnapshotInfo\x12-\n\ractor_volumes\x18\x07 \x03(\x0b\x32\x16.ateapi.ExternalVolume\x12\'\n\x1fin_progress_local_snapshot_name\x18\x08 \x01(\t\x12\"\n\x1a\x63urrent_actor_template_uid\x18\x0b \x01(\t\"\xa7\x01\n\x10WorkerAssignment\x12!\n\x06worker\x18\x06 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x18\n\x10worker_namespace\x18\x01 \x01(\t\x12\x13\n\x0bworker_pool\x18\x02 \x01(\t\x12\x12\n\nworker_pod\x18\x03 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x04 \x01(\t\x12\x15\n\rworker_pod_ip\x18\x05 \x01(\t\"\x8f\x01\n\tTagStatus\x12*\n\x08snapshot\x18\x01 \x01(\x0b\x32\x18.ateapi.ExternalSnapshot\x12\x1a\n\x12\x61\x63tor_template_uid\x18\x02 \x01(\t\x12 \n\x18in_progress_snapshot_uri\x18\x03 \x01(\t\x12\x18\n\x10source_actor_uid\x18\x04 \x01(\t\"\x9e\x01\n\x03Tag\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12!\n\x06status\x18\x02 \x01(\x0b\x32\x11.ateapi.TagStatus\x12\x1f\n\x05scope\x18\x03 \x01(\x0e\x32\x10.ateapi.TagScope\x12\'\n\x0csource_actor\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\"6\n\x08\x41tespace\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\"+\n\tObjectRef\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"\xe3\x02\n\rActorTemplate\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12)\n\x0fworker_selector\x18\x02 \x01(\x0b\x32\x10.ateapi.Selector\x12%\n\ncontainers\x18\x03 \x03(\x0b\x32\x11.ateapi.Container\x12\x1f\n\x07volumes\x18\x04 \x03(\x0b\x32\x0e.ateapi.Volume\x12\x31\n\x10snapshots_config\x18\x05 \x01(\x0b\x32\x17.ateapi.SnapshotsConfig\x12-\n\x0esandbox_config\x18\x06 \x01(\x0b\x32\x15.ateapi.SandboxConfig\x12$\n\tresources\x18\x07 \x01(\x0b\x32\x11.ateapi.Resources\x12+\n\x06status\x18\x08 \x01(\x0b\x32\x1b.ateapi.ActorTemplateStatus\"+\n\tResources\x12\x1e\n\x06limits\x18\x01 \x03(\x0b\x32\x0e.ateapi.Limits\"(\n\x06Limits\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08quantity\x18\x02 \x01(\t\"\x9d\x01\n\x14GoldenSnapshotStatus\x12\x31\n\x0fgolden_snapshot\x18\x01 \x01(\x0b\x32\x18.ateapi.ExternalSnapshot\x12;\n\x17take_golden_snapshot_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x15\n\rerror_message\x18\x03 \x01(\t\"S\n\x13\x41\x63torTemplateStatus\x12<\n\x16golden_snapshot_status\x18\x01 \x01(\x0b\x32\x1c.ateapi.GoldenSnapshotStatus\"Q\n\rSandboxConfig\x12+\n\rsandbox_class\x18\x01 \x01(\x0e\x32\x14.ateapi.SandboxClass\x12\x13\n\x0b\x63onfig_name\x18\x02 \x01(\t\"\xb7\x01\n\x0fSnapshotsConfig\x12.\n\x08on_pause\x18\x01 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12/\n\ton_commit\x18\x02 \x01(\x0e\x32\x1c.ateapi.SnapshotContentScope\x12)\n\ton_resume\x18\x03 \x01(\x0b\x32\x16.ateapi.OnResumeConfig\x12\x18\n\x10storage_location\x18\x04 \x01(\t\"9\n\x0eOnResumeConfig\x12\'\n\tfrom_data\x18\x01 \x01(\x0e\x32\x14.ateapi.ResumeSource\"\x92\x02\n\tContainer\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05image\x18\x02 \x01(\t\x12\x0f\n\x07\x63ommand\x18\x03 \x03(\t\x12\x0c\n\x04\x61rgs\x18\x04 \x03(\t\x12\x1b\n\x03\x65nv\x18\x05 \x03(\x0b\x32\x0e.ateapi.EnvVar\x12\'\n\x06readyz\x18\x06 \x01(\x0b\x32\x17.ateapi.ContainerReadyz\x12*\n\rvolume_mounts\x18\x07 \x03(\x0b\x32\x13.ateapi.VolumeMount\x12\x31\n\x10security_context\x18\x08 \x01(\x0b\x32\x17.ateapi.SecurityContext\x12$\n\tresources\x18\t \x01(\x0b\x32\x11.ateapi.Resources\"=\n\x0fSecurityContext\x12*\n\x0c\x63\x61pabilities\x18\x01 \x01(\x0b\x32\x14.ateapi.Capabilities\")\n\x0c\x43\x61pabilities\x12\x0b\n\x03\x61\x64\x64\x18\x01 \x03(\t\x12\x0c\n\x04\x64rop\x18\x02 \x03(\t\"%\n\x06\x45nvVar\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"S\n\x0f\x43ontainerReadyz\x12\'\n\x08http_get\x18\x01 \x01(\x0b\x32\x15.ateapi.HTTPGetAction\x12\x17\n\x0ftimeout_seconds\x18\x02 \x01(\x05\"+\n\rHTTPGetAction\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x0c\n\x04port\x18\x02 \x01(\x05\"\xec\x01\n\x06Volume\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x33\n\x0b\x64urable_dir\x18\x02 \x01(\x0b\x32\x1e.ateapi.DurableDirVolumeSource\x12@\n\x18\x65xternal_volume_template\x18\x03 \x01(\x0b\x32\x1e.ateapi.ExternalVolumeTemplate\x12\x33\n\x0bsystem_info\x18\x05 \x01(\x0b\x32\x1e.ateapi.SystemInfoVolumeSource\x12(\n\x05image\x18\x06 \x01(\x0b\x32\x19.ateapi.ImageVolumeSource\"&\n\x11ImageVolumeSource\x12\x11\n\treference\x18\x01 \x01(\t\"\x18\n\x16\x44urableDirVolumeSource\"F\n\x16\x45xternalVolumeTemplate\x12\x10\n\x08\x63\x61pacity\x18\x01 \x01(\t\x12\x1a\n\x12storage_class_name\x18\x02 \x01(\t\"L\n\x16SystemInfoVolumeSource\x12\x32\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x1c.ateapi.SystemInfoDataSource\"\x84\x01\n\x14SystemInfoDataSource\x12\x37\n\x0e\x61\x63tor_metadata\x18\x01 \x01(\x0b\x32\x1f.ateapi.ActorMetadataDataSource\x12\x33\n\x0ctrust_bundle\x18\x02 \x01(\x0b\x32\x1d.ateapi.TrustBundleDataSource\"C\n\x17\x41\x63torMetadataDataSource\x12(\n\x05items\x18\x01 \x03(\x0b\x32\x19.ateapi.ActorMetadataItem\"L\n\x11\x41\x63torMetadataItem\x12)\n\x05\x66ield\x18\x01 \x01(\x0e\x32\x1a.ateapi.ActorMetadataField\x12\x0c\n\x04path\x18\x02 \x01(\t\"3\n\x15TrustBundleDataSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\"/\n\x0bVolumeMount\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\nmount_path\x18\x02 \x01(\t\";\n\x15\x43reateAtespaceRequest\x12\"\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x10.ateapi.Atespace\"9\n\x12GetAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"=\n\x14ListAtespacesRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"U\n\x15ListAtespacesResponse\x12#\n\tatespaces\x18\x01 \x03(\x0b\x32\x10.ateapi.Atespace\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"<\n\x15\x44\x65leteAtespaceRequest\x12#\n\x08\x61tespace\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"K\n\x1a\x43reateActorTemplateRequest\x12-\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x15.ateapi.ActorTemplate\"D\n\x17GetActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"T\n\x19ListActorTemplatesRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"e\n\x1aListActorTemplatesResponse\x12.\n\x0f\x61\x63tor_templates\x18\x01 \x03(\x0b\x32\x15.ateapi.ActorTemplate\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"G\n\x1a\x44\x65leteActorTemplateRequest\x12)\n\x0e\x61\x63tor_template\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"3\n\x0fGetActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12\x43reateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"2\n\x12UpdateActorRequest\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"7\n\x13SuspendActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"4\n\x14SuspendActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"c\n\x1cSuspendActorWithLeaseRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12!\n\x05lease\x18\x02 \x01(\x0b\x32\x12.ateapi.ActorLease\"5\n\x11PauseActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"2\n\x12PauseActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\"\x84\x01\n\x12ResumeActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x0c\n\x04\x62oot\x18\x02 \x01(\x08\x12!\n\x05lease\x18\x03 \x01(\x0b\x32\x12.ateapi.ActorLease\x12\x1b\n\x13\x63laim_runtime_lease\x18\x04 \x01(\x08\"g\n\x13ResumeActorResponse\x12\x1c\n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\r.ateapi.Actor\x12\x0f\n\x07resumed\x18\x02 \x01(\x08\x12!\n\x05lease\x18\x03 \x01(\x0b\x32\x12.ateapi.ActorLease\"_\n\nActorLease\x12\r\n\x05token\x18\x01 \x01(\t\x12\x12\n\ngeneration\x18\x02 \x01(\x03\x12.\n\nexpires_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"]\n\x16RenewActorLeaseRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12!\n\x05lease\x18\x02 \x01(\x0b\x32\x12.ateapi.ActorLease\"<\n\x17RenewActorLeaseResponse\x12!\n\x05lease\x18\x01 \x01(\x0b\x32\x12.ateapi.ActorLease\"I\n\x12\x44\x65leteActorRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tany_state\x18\x02 \x01(\x08\"?\n\x1bGetActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"o\n\x1e\x43reateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"o\n\x1eUpdateActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12+\n\regress_policy\x18\x02 \x01(\x0b\x32\x14.ateapi.EgressPolicy\"B\n\x1e\x44\x65leteActorEgressPolicyRequest\x12 \n\x05\x61\x63tor\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"/\n\rGetTagRequest\x12\x1e\n\x03tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"J\n\x0fListTagsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"F\n\x10ListTagsResponse\x12\x19\n\x04tags\x18\x01 \x03(\x0b\x32\x0b.ateapi.Tag\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\",\n\x10\x43reateTagRequest\x12\x18\n\x03tag\x18\x01 \x01(\x0b\x32\x0b.ateapi.Tag\",\n\x10UpdateTagRequest\x12\x18\n\x03tag\x18\x01 \x01(\x0b\x32\x0b.ateapi.Tag\"2\n\x10\x44\x65leteTagRequest\x12\x1e\n\x03tag\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"-\n\rDeleteOptions\x12\x0f\n\x07version\x18\x01 \x01(\x03\x12\x0b\n\x03uid\x18\x02 \x01(\t\"m\n!ListWorkerActorAssignmentsRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"q\n\"ListWorkerActorAssignmentsResponse\x12\x32\n\x11\x61\x63tor_assignments\x18\x01 \x03(\x0b\x32\x17.ateapi.ActorAssignment\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\";\n\x12ListWorkersRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\"O\n\x13ListWorkersResponse\x12\x1f\n\x07workers\x18\x01 \x03(\x0b\x32\x0e.ateapi.Worker\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"5\n\x10GetWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"5\n\x13\x43reateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"5\n\x13UpdateWorkerRequest\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"`\n\x13\x44\x65leteWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12&\n\x07options\x18\x02 \x01(\x0b\x32\x15.ateapi.DeleteOptions\"7\n\x12\x44rainWorkerRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\"L\n\x11ListActorsRequest\x12\x10\n\x08\x61tespace\x18\x01 \x01(\t\x12\x11\n\tpage_size\x18\x02 \x01(\x05\x12\x12\n\npage_token\x18\x03 \x01(\t\"L\n\x12ListActorsResponse\x12\x1d\n\x06\x61\x63tors\x18\x01 \x03(\x0b\x32\r.ateapi.Actor\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"\xc6\x02\n\x06Worker\x12*\n\x08metadata\x18\x01 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12\x18\n\x10worker_namespace\x18\x02 \x01(\t\x12\x13\n\x0bworker_pool\x18\x03 \x01(\t\x12\x12\n\nworker_pod\x18\x04 \x01(\t\x12\x16\n\x0eworker_pod_uid\x18\x05 \x01(\t\x12\x11\n\tnode_name\x18\x06 \x01(\t\x12\n\n\x02ip\x18\x07 \x01(\t\x12\x15\n\rsandbox_class\x18\x08 \x01(\t\x12*\n\x06labels\x18\t \x03(\x0b\x32\x1a.ateapi.Worker.LabelsEntry\x12$\n\x06status\x18\x0b \x01(\x0b\x32\x14.ateapi.WorkerStatus\x1a-\n\x0bLabelsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x89\x01\n\x0cWorkerStatus\x12\"\n\x05state\x18\x01 \x01(\x0e\x32\x13.ateapi.WorkerState\x12)\n\x08\x63\x61pacity\x18\x02 \x01(\x0b\x32\x17.ateapi.WorkerResources\x12*\n\tallocated\x18\x03 \x01(\x0b\x32\x17.ateapi.WorkerResources\"G\n\x0fWorkerResources\x12$\n\tresources\x18\x01 \x01(\x0b\x32\x11.ateapi.Resources\x12\x0e\n\x06\x61\x63tors\x18\x02 \x01(\x05\"\xc7\x01\n\x0f\x41\x63torAssignment\x12*\n\x08metadata\x18\x06 \x01(\x0b\x32\x18.ateapi.ResourceMetadata\x12 \n\x05\x61\x63tor\x18\x02 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12\x11\n\tactor_uid\x18\x03 \x01(\t\x12-\n\x12\x61\x63tor_template_ref\x18\x04 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12$\n\tresources\x18\x05 \x01(\x0b\x32\x11.ateapi.Resources\"h\n\x18SetWorkerCapacityRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12)\n\x08\x63\x61pacity\x18\x02 \x01(\x0b\x32\x17.ateapi.WorkerResources\";\n\x19SetWorkerCapacityResponse\x12\x1e\n\x06worker\x18\x01 \x01(\x0b\x32\x0e.ateapi.Worker\"[\n\x0eMintJWTRequest\x12\x10\n\x08\x61udience\x18\x01 \x03(\t\x12\x10\n\x08\x61tespace\x18\x02 \x01(\t\x12\x12\n\nactor_name\x18\x03 \x01(\t\x12\x11\n\tactor_uid\x18\x04 \x01(\t\"$\n\x0fMintJWTResponse\x12\x11\n\tactor_jwt\x18\x01 \x01(\t\"\xa7\x01\n\x0fMintCertRequest\x12!\n\x06worker\x18\x01 \x01(\x0b\x32\x11.ateapi.ObjectRef\x12#\n\x1b\x63\x65rtificate_signing_request\x18\x02 \x01(\x0c\x12\x1a\n\x12\x65xpected_actor_uid\x18\x03 \x01(\t\x12\x30\n\x07purpose\x18\x04 \x01(\x0e\x32\x1f.ateapi.ActorCertificatePurpose\".\n\x10MintCertResponse\x12\x1a\n\x12\x61\x63tor_certificates\x18\x01 \x03(\x0c*\x80\x01\n\x14SnapshotContentScope\x12&\n\"SNAPSHOT_CONTENT_SCOPE_UNSPECIFIED\x10\x00\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_FULL\x10\x01\x12\x1f\n\x1bSNAPSHOT_CONTENT_SCOPE_DATA\x10\x02*V\n\x08TagScope\x12\x19\n\x15TAG_SCOPE_UNSPECIFIED\x10\x00\x12\x16\n\x12TAG_SCOPE_ATESPACE\x10\x01\x12\x17\n\x13TAG_SCOPE_PUBLISHED\x10\x02*\xf7\x01\n\nActorState\x12\x1b\n\x17\x41\x43TOR_STATE_UNSPECIFIED\x10\x00\x12\x18\n\x14\x41\x43TOR_STATE_RESUMING\x10\x01\x12\x17\n\x13\x41\x43TOR_STATE_RUNNING\x10\x02\x12\x1a\n\x16\x41\x43TOR_STATE_SUSPENDING\x10\x03\x12\x19\n\x15\x41\x43TOR_STATE_SUSPENDED\x10\x04\x12\x17\n\x13\x41\x43TOR_STATE_PAUSING\x10\x05\x12\x16\n\x12\x41\x43TOR_STATE_PAUSED\x10\x06\x12\x17\n\x13\x41\x43TOR_STATE_CRASHED\x10\x07\x12\x18\n\x14\x41\x43TOR_STATE_DELETING\x10\x08*b\n\x0cSandboxClass\x12\x1d\n\x19SANDBOX_CLASS_UNSPECIFIED\x10\x00\x12\x18\n\x14SANDBOX_CLASS_GVISOR\x10\x01\x12\x19\n\x15SANDBOX_CLASS_MICROVM\x10\x02*d\n\x0cResumeSource\x12\x1d\n\x19RESUME_SOURCE_UNSPECIFIED\x10\x00\x12\x1b\n\x17RESUME_SOURCE_COLD_BOOT\x10\x01\x12\x18\n\x14RESUME_SOURCE_GOLDEN\x10\x02*\x9a\x01\n\x12\x41\x63torMetadataField\x12$\n ACTOR_METADATA_FIELD_UNSPECIFIED\x10\x00\x12\x1d\n\x19\x41\x43TOR_METADATA_FIELD_NAME\x10\x01\x12!\n\x1d\x41\x43TOR_METADATA_FIELD_ATESPACE\x10\x02\x12\x1c\n\x18\x41\x43TOR_METADATA_FIELD_UID\x10\x03*_\n\x0bWorkerState\x12\x1c\n\x18WORKER_STATE_UNSPECIFIED\x10\x00\x12\x17\n\x13WORKER_STATE_ACTIVE\x10\x01\x12\x19\n\x15WORKER_STATE_DRAINING\x10\x02*k\n\x17\x41\x63torCertificatePurpose\x12)\n%ACTOR_CERTIFICATE_PURPOSE_UNSPECIFIED\x10\x00\x12%\n!ACTOR_CERTIFICATE_PURPOSE_ATUNNEL\x10\x01\x32\xa8\x13\n\x07\x43ontrol\x12\x34\n\x08GetActor\x12\x17.ateapi.GetActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0b\x43reateActor\x12\x1a.ateapi.CreateActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n\x0bUpdateActor\x12\x1a.ateapi.UpdateActorRequest\x1a\r.ateapi.Actor\"\x00\x12K\n\x0cSuspendActor\x12\x1b.ateapi.SuspendActorRequest\x1a\x1c.ateapi.SuspendActorResponse\"\x00\x12]\n\x15SuspendActorWithLease\x12$.ateapi.SuspendActorWithLeaseRequest\x1a\x1c.ateapi.SuspendActorResponse\"\x00\x12\x45\n\nPauseActor\x12\x19.ateapi.PauseActorRequest\x1a\x1a.ateapi.PauseActorResponse\"\x00\x12H\n\x0bResumeActor\x12\x1a.ateapi.ResumeActorRequest\x1a\x1b.ateapi.ResumeActorResponse\"\x00\x12T\n\x0fRenewActorLease\x12\x1e.ateapi.RenewActorLeaseRequest\x1a\x1f.ateapi.RenewActorLeaseResponse\"\x00\x12:\n\x0b\x44\x65leteActor\x12\x1a.ateapi.DeleteActorRequest\x1a\r.ateapi.Actor\"\x00\x12S\n\x14GetActorEgressPolicy\x12#.ateapi.GetActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x43reateActorEgressPolicy\x12&.ateapi.CreateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17UpdateActorEgressPolicy\x12&.ateapi.UpdateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n\x17\x44\x65leteActorEgressPolicy\x12&.ateapi.DeleteActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12\x34\n\tCreateTag\x12\x18.ateapi.CreateTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12.\n\x06GetTag\x12\x15.ateapi.GetTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12?\n\x08ListTags\x12\x17.ateapi.ListTagsRequest\x1a\x18.ateapi.ListTagsResponse\"\x00\x12\x34\n\tUpdateTag\x12\x18.ateapi.UpdateTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12\x34\n\tDeleteTag\x12\x18.ateapi.DeleteTagRequest\x1a\x0b.ateapi.Tag\"\x00\x12H\n\x0bListWorkers\x12\x1a.ateapi.ListWorkersRequest\x1a\x1b.ateapi.ListWorkersResponse\"\x00\x12\x37\n\tGetWorker\x12\x18.ateapi.GetWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x43reateWorker\x12\x1b.ateapi.CreateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0cUpdateWorker\x12\x1b.ateapi.UpdateWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12=\n\x0c\x44\x65leteWorker\x12\x1b.ateapi.DeleteWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12;\n\x0b\x44rainWorker\x12\x1a.ateapi.DrainWorkerRequest\x1a\x0e.ateapi.Worker\"\x00\x12u\n\x1aListWorkerActorAssignments\x12).ateapi.ListWorkerActorAssignmentsRequest\x1a*.ateapi.ListWorkerActorAssignmentsResponse\"\x00\x12\x45\n\nListActors\x12\x19.ateapi.ListActorsRequest\x1a\x1a.ateapi.ListActorsResponse\"\x00\x12\x43\n\x0e\x43reateAtespace\x12\x1d.ateapi.CreateAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12=\n\x0bGetAtespace\x12\x1a.ateapi.GetAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12N\n\rListAtespaces\x12\x1c.ateapi.ListAtespacesRequest\x1a\x1d.ateapi.ListAtespacesResponse\"\x00\x12\x43\n\x0e\x44\x65leteAtespace\x12\x1d.ateapi.DeleteAtespaceRequest\x1a\x10.ateapi.Atespace\"\x00\x12R\n\x13\x43reateActorTemplate\x12\".ateapi.CreateActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12L\n\x10GetActorTemplate\x12\x1f.ateapi.GetActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x12]\n\x12ListActorTemplates\x12!.ateapi.ListActorTemplatesRequest\x1a\".ateapi.ListActorTemplatesResponse\"\x00\x12R\n\x13\x44\x65leteActorTemplate\x12\".ateapi.DeleteActorTemplateRequest\x1a\x15.ateapi.ActorTemplate\"\x00\x32\x8a\x01\n\rActorIdentity\x12:\n\x07MintJWT\x12\x16.ateapi.MintJWTRequest\x1a\x17.ateapi.MintJWTResponse\x12=\n\x08MintCert\x12\x17.ateapi.MintCertRequest\x1a\x18.ateapi.MintCertResponse2i\n\rWorkerService\x12X\n\x11SetWorkerCapacity\x12 .ateapi.SetWorkerCapacityRequest\x1a!.ateapi.SetWorkerCapacityResponseB9Z7github.com/agent-substrate/substrate/pkg/proto/ateapipbb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -54,22 +54,22 @@ _globals['_EXTERNALVOLUME_VOLUMECONTEXTENTRY']._serialized_options = b'8\001' _globals['_WORKER_LABELSENTRY']._loaded_options = None _globals['_WORKER_LABELSENTRY']._serialized_options = b'8\001' - _globals['_SNAPSHOTCONTENTSCOPE']._serialized_start=9237 - _globals['_SNAPSHOTCONTENTSCOPE']._serialized_end=9365 - _globals['_TAGSCOPE']._serialized_start=9367 - _globals['_TAGSCOPE']._serialized_end=9453 - _globals['_ACTORSTATE']._serialized_start=9456 - _globals['_ACTORSTATE']._serialized_end=9703 - _globals['_SANDBOXCLASS']._serialized_start=9705 - _globals['_SANDBOXCLASS']._serialized_end=9803 - _globals['_RESUMESOURCE']._serialized_start=9805 - _globals['_RESUMESOURCE']._serialized_end=9905 - _globals['_ACTORMETADATAFIELD']._serialized_start=9908 - _globals['_ACTORMETADATAFIELD']._serialized_end=10062 - _globals['_WORKERSTATE']._serialized_start=10064 - _globals['_WORKERSTATE']._serialized_end=10159 - _globals['_ACTORCERTIFICATEPURPOSE']._serialized_start=10161 - _globals['_ACTORCERTIFICATEPURPOSE']._serialized_end=10268 + _globals['_SNAPSHOTCONTENTSCOPE']._serialized_start=9692 + _globals['_SNAPSHOTCONTENTSCOPE']._serialized_end=9820 + _globals['_TAGSCOPE']._serialized_start=9822 + _globals['_TAGSCOPE']._serialized_end=9908 + _globals['_ACTORSTATE']._serialized_start=9911 + _globals['_ACTORSTATE']._serialized_end=10158 + _globals['_SANDBOXCLASS']._serialized_start=10160 + _globals['_SANDBOXCLASS']._serialized_end=10258 + _globals['_RESUMESOURCE']._serialized_start=10260 + _globals['_RESUMESOURCE']._serialized_end=10360 + _globals['_ACTORMETADATAFIELD']._serialized_start=10363 + _globals['_ACTORMETADATAFIELD']._serialized_end=10517 + _globals['_WORKERSTATE']._serialized_start=10519 + _globals['_WORKERSTATE']._serialized_end=10614 + _globals['_ACTORCERTIFICATEPURPOSE']._serialized_start=10616 + _globals['_ACTORCERTIFICATEPURPOSE']._serialized_end=10723 _globals['_EXTERNALSNAPSHOT']._serialized_start=86 _globals['_EXTERNALSNAPSHOT']._serialized_end=179 _globals['_LOCALSNAPSHOTINFO']._serialized_start=182 @@ -190,86 +190,94 @@ _globals['_SUSPENDACTORREQUEST']._serialized_end=6007 _globals['_SUSPENDACTORRESPONSE']._serialized_start=6009 _globals['_SUSPENDACTORRESPONSE']._serialized_end=6061 - _globals['_PAUSEACTORREQUEST']._serialized_start=6063 - _globals['_PAUSEACTORREQUEST']._serialized_end=6116 - _globals['_PAUSEACTORRESPONSE']._serialized_start=6118 - _globals['_PAUSEACTORRESPONSE']._serialized_end=6168 - _globals['_RESUMEACTORREQUEST']._serialized_start=6170 - _globals['_RESUMEACTORREQUEST']._serialized_end=6238 - _globals['_RESUMEACTORRESPONSE']._serialized_start=6240 - _globals['_RESUMEACTORRESPONSE']._serialized_end=6308 - _globals['_DELETEACTORREQUEST']._serialized_start=6310 - _globals['_DELETEACTORREQUEST']._serialized_end=6383 - _globals['_GETACTOREGRESSPOLICYREQUEST']._serialized_start=6385 - _globals['_GETACTOREGRESSPOLICYREQUEST']._serialized_end=6448 - _globals['_CREATEACTOREGRESSPOLICYREQUEST']._serialized_start=6450 - _globals['_CREATEACTOREGRESSPOLICYREQUEST']._serialized_end=6561 - _globals['_UPDATEACTOREGRESSPOLICYREQUEST']._serialized_start=6563 - _globals['_UPDATEACTOREGRESSPOLICYREQUEST']._serialized_end=6674 - _globals['_DELETEACTOREGRESSPOLICYREQUEST']._serialized_start=6676 - _globals['_DELETEACTOREGRESSPOLICYREQUEST']._serialized_end=6742 - _globals['_GETTAGREQUEST']._serialized_start=6744 - _globals['_GETTAGREQUEST']._serialized_end=6791 - _globals['_LISTTAGSREQUEST']._serialized_start=6793 - _globals['_LISTTAGSREQUEST']._serialized_end=6867 - _globals['_LISTTAGSRESPONSE']._serialized_start=6869 - _globals['_LISTTAGSRESPONSE']._serialized_end=6939 - _globals['_CREATETAGREQUEST']._serialized_start=6941 - _globals['_CREATETAGREQUEST']._serialized_end=6985 - _globals['_UPDATETAGREQUEST']._serialized_start=6987 - _globals['_UPDATETAGREQUEST']._serialized_end=7031 - _globals['_DELETETAGREQUEST']._serialized_start=7033 - _globals['_DELETETAGREQUEST']._serialized_end=7083 - _globals['_DELETEOPTIONS']._serialized_start=7085 - _globals['_DELETEOPTIONS']._serialized_end=7130 - _globals['_LISTWORKERACTORASSIGNMENTSREQUEST']._serialized_start=7132 - _globals['_LISTWORKERACTORASSIGNMENTSREQUEST']._serialized_end=7241 - _globals['_LISTWORKERACTORASSIGNMENTSRESPONSE']._serialized_start=7243 - _globals['_LISTWORKERACTORASSIGNMENTSRESPONSE']._serialized_end=7356 - _globals['_LISTWORKERSREQUEST']._serialized_start=7358 - _globals['_LISTWORKERSREQUEST']._serialized_end=7417 - _globals['_LISTWORKERSRESPONSE']._serialized_start=7419 - _globals['_LISTWORKERSRESPONSE']._serialized_end=7498 - _globals['_GETWORKERREQUEST']._serialized_start=7500 - _globals['_GETWORKERREQUEST']._serialized_end=7553 - _globals['_CREATEWORKERREQUEST']._serialized_start=7555 - _globals['_CREATEWORKERREQUEST']._serialized_end=7608 - _globals['_UPDATEWORKERREQUEST']._serialized_start=7610 - _globals['_UPDATEWORKERREQUEST']._serialized_end=7663 - _globals['_DELETEWORKERREQUEST']._serialized_start=7665 - _globals['_DELETEWORKERREQUEST']._serialized_end=7761 - _globals['_DRAINWORKERREQUEST']._serialized_start=7763 - _globals['_DRAINWORKERREQUEST']._serialized_end=7818 - _globals['_LISTACTORSREQUEST']._serialized_start=7820 - _globals['_LISTACTORSREQUEST']._serialized_end=7896 - _globals['_LISTACTORSRESPONSE']._serialized_start=7898 - _globals['_LISTACTORSRESPONSE']._serialized_end=7974 - _globals['_WORKER']._serialized_start=7977 - _globals['_WORKER']._serialized_end=8303 - _globals['_WORKER_LABELSENTRY']._serialized_start=8258 - _globals['_WORKER_LABELSENTRY']._serialized_end=8303 - _globals['_WORKERSTATUS']._serialized_start=8306 - _globals['_WORKERSTATUS']._serialized_end=8443 - _globals['_WORKERRESOURCES']._serialized_start=8445 - _globals['_WORKERRESOURCES']._serialized_end=8516 - _globals['_ACTORASSIGNMENT']._serialized_start=8519 - _globals['_ACTORASSIGNMENT']._serialized_end=8718 - _globals['_SETWORKERCAPACITYREQUEST']._serialized_start=8720 - _globals['_SETWORKERCAPACITYREQUEST']._serialized_end=8824 - _globals['_SETWORKERCAPACITYRESPONSE']._serialized_start=8826 - _globals['_SETWORKERCAPACITYRESPONSE']._serialized_end=8885 - _globals['_MINTJWTREQUEST']._serialized_start=8887 - _globals['_MINTJWTREQUEST']._serialized_end=8978 - _globals['_MINTJWTRESPONSE']._serialized_start=8980 - _globals['_MINTJWTRESPONSE']._serialized_end=9016 - _globals['_MINTCERTREQUEST']._serialized_start=9019 - _globals['_MINTCERTREQUEST']._serialized_end=9186 - _globals['_MINTCERTRESPONSE']._serialized_start=9188 - _globals['_MINTCERTRESPONSE']._serialized_end=9234 - _globals['_CONTROL']._serialized_start=10271 - _globals['_CONTROL']._serialized_end=12562 - _globals['_ACTORIDENTITY']._serialized_start=12565 - _globals['_ACTORIDENTITY']._serialized_end=12703 - _globals['_WORKERSERVICE']._serialized_start=12705 - _globals['_WORKERSERVICE']._serialized_end=12810 + _globals['_SUSPENDACTORWITHLEASEREQUEST']._serialized_start=6063 + _globals['_SUSPENDACTORWITHLEASEREQUEST']._serialized_end=6162 + _globals['_PAUSEACTORREQUEST']._serialized_start=6164 + _globals['_PAUSEACTORREQUEST']._serialized_end=6217 + _globals['_PAUSEACTORRESPONSE']._serialized_start=6219 + _globals['_PAUSEACTORRESPONSE']._serialized_end=6269 + _globals['_RESUMEACTORREQUEST']._serialized_start=6272 + _globals['_RESUMEACTORREQUEST']._serialized_end=6404 + _globals['_RESUMEACTORRESPONSE']._serialized_start=6406 + _globals['_RESUMEACTORRESPONSE']._serialized_end=6509 + _globals['_ACTORLEASE']._serialized_start=6511 + _globals['_ACTORLEASE']._serialized_end=6606 + _globals['_RENEWACTORLEASEREQUEST']._serialized_start=6608 + _globals['_RENEWACTORLEASEREQUEST']._serialized_end=6701 + _globals['_RENEWACTORLEASERESPONSE']._serialized_start=6703 + _globals['_RENEWACTORLEASERESPONSE']._serialized_end=6763 + _globals['_DELETEACTORREQUEST']._serialized_start=6765 + _globals['_DELETEACTORREQUEST']._serialized_end=6838 + _globals['_GETACTOREGRESSPOLICYREQUEST']._serialized_start=6840 + _globals['_GETACTOREGRESSPOLICYREQUEST']._serialized_end=6903 + _globals['_CREATEACTOREGRESSPOLICYREQUEST']._serialized_start=6905 + _globals['_CREATEACTOREGRESSPOLICYREQUEST']._serialized_end=7016 + _globals['_UPDATEACTOREGRESSPOLICYREQUEST']._serialized_start=7018 + _globals['_UPDATEACTOREGRESSPOLICYREQUEST']._serialized_end=7129 + _globals['_DELETEACTOREGRESSPOLICYREQUEST']._serialized_start=7131 + _globals['_DELETEACTOREGRESSPOLICYREQUEST']._serialized_end=7197 + _globals['_GETTAGREQUEST']._serialized_start=7199 + _globals['_GETTAGREQUEST']._serialized_end=7246 + _globals['_LISTTAGSREQUEST']._serialized_start=7248 + _globals['_LISTTAGSREQUEST']._serialized_end=7322 + _globals['_LISTTAGSRESPONSE']._serialized_start=7324 + _globals['_LISTTAGSRESPONSE']._serialized_end=7394 + _globals['_CREATETAGREQUEST']._serialized_start=7396 + _globals['_CREATETAGREQUEST']._serialized_end=7440 + _globals['_UPDATETAGREQUEST']._serialized_start=7442 + _globals['_UPDATETAGREQUEST']._serialized_end=7486 + _globals['_DELETETAGREQUEST']._serialized_start=7488 + _globals['_DELETETAGREQUEST']._serialized_end=7538 + _globals['_DELETEOPTIONS']._serialized_start=7540 + _globals['_DELETEOPTIONS']._serialized_end=7585 + _globals['_LISTWORKERACTORASSIGNMENTSREQUEST']._serialized_start=7587 + _globals['_LISTWORKERACTORASSIGNMENTSREQUEST']._serialized_end=7696 + _globals['_LISTWORKERACTORASSIGNMENTSRESPONSE']._serialized_start=7698 + _globals['_LISTWORKERACTORASSIGNMENTSRESPONSE']._serialized_end=7811 + _globals['_LISTWORKERSREQUEST']._serialized_start=7813 + _globals['_LISTWORKERSREQUEST']._serialized_end=7872 + _globals['_LISTWORKERSRESPONSE']._serialized_start=7874 + _globals['_LISTWORKERSRESPONSE']._serialized_end=7953 + _globals['_GETWORKERREQUEST']._serialized_start=7955 + _globals['_GETWORKERREQUEST']._serialized_end=8008 + _globals['_CREATEWORKERREQUEST']._serialized_start=8010 + _globals['_CREATEWORKERREQUEST']._serialized_end=8063 + _globals['_UPDATEWORKERREQUEST']._serialized_start=8065 + _globals['_UPDATEWORKERREQUEST']._serialized_end=8118 + _globals['_DELETEWORKERREQUEST']._serialized_start=8120 + _globals['_DELETEWORKERREQUEST']._serialized_end=8216 + _globals['_DRAINWORKERREQUEST']._serialized_start=8218 + _globals['_DRAINWORKERREQUEST']._serialized_end=8273 + _globals['_LISTACTORSREQUEST']._serialized_start=8275 + _globals['_LISTACTORSREQUEST']._serialized_end=8351 + _globals['_LISTACTORSRESPONSE']._serialized_start=8353 + _globals['_LISTACTORSRESPONSE']._serialized_end=8429 + _globals['_WORKER']._serialized_start=8432 + _globals['_WORKER']._serialized_end=8758 + _globals['_WORKER_LABELSENTRY']._serialized_start=8713 + _globals['_WORKER_LABELSENTRY']._serialized_end=8758 + _globals['_WORKERSTATUS']._serialized_start=8761 + _globals['_WORKERSTATUS']._serialized_end=8898 + _globals['_WORKERRESOURCES']._serialized_start=8900 + _globals['_WORKERRESOURCES']._serialized_end=8971 + _globals['_ACTORASSIGNMENT']._serialized_start=8974 + _globals['_ACTORASSIGNMENT']._serialized_end=9173 + _globals['_SETWORKERCAPACITYREQUEST']._serialized_start=9175 + _globals['_SETWORKERCAPACITYREQUEST']._serialized_end=9279 + _globals['_SETWORKERCAPACITYRESPONSE']._serialized_start=9281 + _globals['_SETWORKERCAPACITYRESPONSE']._serialized_end=9340 + _globals['_MINTJWTREQUEST']._serialized_start=9342 + _globals['_MINTJWTREQUEST']._serialized_end=9433 + _globals['_MINTJWTRESPONSE']._serialized_start=9435 + _globals['_MINTJWTRESPONSE']._serialized_end=9471 + _globals['_MINTCERTREQUEST']._serialized_start=9474 + _globals['_MINTCERTREQUEST']._serialized_end=9641 + _globals['_MINTCERTRESPONSE']._serialized_start=9643 + _globals['_MINTCERTRESPONSE']._serialized_end=9689 + _globals['_CONTROL']._serialized_start=10726 + _globals['_CONTROL']._serialized_end=13198 + _globals['_ACTORIDENTITY']._serialized_start=13201 + _globals['_ACTORIDENTITY']._serialized_end=13339 + _globals['_WORKERSERVICE']._serialized_start=13341 + _globals['_WORKERSERVICE']._serialized_end=13446 # @@protoc_insertion_point(module_scope) diff --git a/benchmarking/locust/common/ateapi_pb2_grpc.py b/benchmarking/locust/common/ateapi_pb2_grpc.py index 069ac3e2eb..e4e4356076 100644 --- a/benchmarking/locust/common/ateapi_pb2_grpc.py +++ b/benchmarking/locust/common/ateapi_pb2_grpc.py @@ -69,6 +69,11 @@ def __init__(self, channel): request_serializer=ateapi__pb2.SuspendActorRequest.SerializeToString, response_deserializer=ateapi__pb2.SuspendActorResponse.FromString, _registered_method=True) + self.SuspendActorWithLease = channel.unary_unary( + '/ateapi.Control/SuspendActorWithLease', + request_serializer=ateapi__pb2.SuspendActorWithLeaseRequest.SerializeToString, + response_deserializer=ateapi__pb2.SuspendActorResponse.FromString, + _registered_method=True) self.PauseActor = channel.unary_unary( '/ateapi.Control/PauseActor', request_serializer=ateapi__pb2.PauseActorRequest.SerializeToString, @@ -79,6 +84,11 @@ def __init__(self, channel): request_serializer=ateapi__pb2.ResumeActorRequest.SerializeToString, response_deserializer=ateapi__pb2.ResumeActorResponse.FromString, _registered_method=True) + self.RenewActorLease = channel.unary_unary( + '/ateapi.Control/RenewActorLease', + request_serializer=ateapi__pb2.RenewActorLeaseRequest.SerializeToString, + response_deserializer=ateapi__pb2.RenewActorLeaseResponse.FromString, + _registered_method=True) self.DeleteActor = channel.unary_unary( '/ateapi.Control/DeleteActor', request_serializer=ateapi__pb2.DeleteActorRequest.SerializeToString, @@ -245,6 +255,13 @@ def SuspendActor(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def SuspendActorWithLease(self, request, context): + """Suspend an actor only when the caller holds its runtime lease. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def PauseActor(self, request, context): """Pause a given actor and keep its snapshots on node VM. """ @@ -259,6 +276,13 @@ def ResumeActor(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def RenewActorLease(self, request, context): + """Renew an actor's runtime lease. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def DeleteActor(self, request, context): """Delete an actor. Only suspended actors can be deleted. """ @@ -470,6 +494,11 @@ def add_ControlServicer_to_server(servicer, server): request_deserializer=ateapi__pb2.SuspendActorRequest.FromString, response_serializer=ateapi__pb2.SuspendActorResponse.SerializeToString, ), + 'SuspendActorWithLease': grpc.unary_unary_rpc_method_handler( + servicer.SuspendActorWithLease, + request_deserializer=ateapi__pb2.SuspendActorWithLeaseRequest.FromString, + response_serializer=ateapi__pb2.SuspendActorResponse.SerializeToString, + ), 'PauseActor': grpc.unary_unary_rpc_method_handler( servicer.PauseActor, request_deserializer=ateapi__pb2.PauseActorRequest.FromString, @@ -480,6 +509,11 @@ def add_ControlServicer_to_server(servicer, server): request_deserializer=ateapi__pb2.ResumeActorRequest.FromString, response_serializer=ateapi__pb2.ResumeActorResponse.SerializeToString, ), + 'RenewActorLease': grpc.unary_unary_rpc_method_handler( + servicer.RenewActorLease, + request_deserializer=ateapi__pb2.RenewActorLeaseRequest.FromString, + response_serializer=ateapi__pb2.RenewActorLeaseResponse.SerializeToString, + ), 'DeleteActor': grpc.unary_unary_rpc_method_handler( servicer.DeleteActor, request_deserializer=ateapi__pb2.DeleteActorRequest.FromString, @@ -730,6 +764,33 @@ def SuspendActor(request, metadata, _registered_method=True) + @staticmethod + def SuspendActorWithLease(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/ateapi.Control/SuspendActorWithLease', + ateapi__pb2.SuspendActorWithLeaseRequest.SerializeToString, + ateapi__pb2.SuspendActorResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def PauseActor(request, target, @@ -784,6 +845,33 @@ def ResumeActor(request, metadata, _registered_method=True) + @staticmethod + def RenewActorLease(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/ateapi.Control/RenewActorLease', + ateapi__pb2.RenewActorLeaseRequest.SerializeToString, + ateapi__pb2.RenewActorLeaseResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) + @staticmethod def DeleteActor(request, target, diff --git a/cmd/ateapi/internal/controlapi/actor.go b/cmd/ateapi/internal/controlapi/actor.go index cf66109d4b..084ebea733 100644 --- a/cmd/ateapi/internal/controlapi/actor.go +++ b/cmd/ateapi/internal/controlapi/actor.go @@ -461,7 +461,15 @@ func (s *RPCService) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorR actorRef := resources.ActorRefFromObjectRef(req.GetActor()) setSpanActorRefAttributes(ctx, actorRef) - actor, resumed, err := s.actorWorkflow.ResumeActor(ctx, actorRef, req.GetBoot()) + var actor *ateapipb.Actor + var resumed bool + var runtimeLease *ateapipb.ActorLease + var err error + if req.GetClaimRuntimeLease() || req.GetLease() != nil { + actor, resumed, runtimeLease, err = s.actorWorkflow.ResumeActorWithLease(ctx, actorRef, req.GetBoot(), req.GetLease()) + } else { + actor, resumed, err = s.actorWorkflow.ResumeActor(ctx, actorRef, req.GetBoot()) + } if err != nil { if errors.Is(err, store.ErrVersionConflict) { return nil, status.Error(codes.Aborted, "concurrent update conflict, please retry") @@ -473,7 +481,18 @@ func (s *RPCService) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorR } setSpanActorAttributes(ctx, actor) - return &ateapipb.ResumeActorResponse{Actor: actor, Resumed: resumed}, nil + return &ateapipb.ResumeActorResponse{Actor: actor, Resumed: resumed, Lease: runtimeLease}, nil +} + +// ResumeActorForReconciler is not part of the public Control contract. It is +// the in-process golden-actor path that may reattach to its persisted lease. +func (s *RPCService) ResumeActorForReconciler(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) { + actorRef := resources.ActorRefFromObjectRef(req.GetActor()) + actor, resumed, runtimeLease, err := s.actorWorkflow.ResumeActorForReconciler(ctx, actorRef, req.GetBoot()) + if err != nil { + return nil, err + } + return &ateapipb.ResumeActorResponse{Actor: actor, Resumed: resumed, Lease: runtimeLease}, nil } func validateResumeActorRequest(ctx context.Context, req *ateapipb.ResumeActorRequest) field.ErrorList { @@ -503,6 +522,76 @@ func (s *RPCService) SuspendActor(ctx context.Context, req *ateapipb.SuspendActo return &ateapipb.SuspendActorResponse{Actor: actor}, nil } +func (s *RPCService) SuspendActorWithLease(ctx context.Context, req *ateapipb.SuspendActorWithLeaseRequest) (*ateapipb.SuspendActorResponse, error) { + if errs := validateSuspendActorWithLeaseRequest(ctx, req); len(errs) > 0 { + return nil, toGRPCStatusError(errs) + } + actorRef := resources.ActorRefFromObjectRef(req.GetActor()) + setSpanActorRefAttributes(ctx, actorRef) + + actor, err := s.actorWorkflow.SuspendActorWithLease(ctx, actorRef, req.GetLease()) + if err != nil { + if errors.Is(err, store.ErrVersionConflict) { + return nil, status.Error(codes.Aborted, "concurrent update conflict, please retry") + } + if errors.Is(err, store.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "Actor %s not found", actorRef) + } + return nil, err + } + setSpanActorAttributes(ctx, actor) + return &ateapipb.SuspendActorResponse{Actor: actor}, nil +} + +// SuspendActorForReconciler is the in-process golden-actor counterpart to +// SuspendActorWithLease; it looks up the persisted lease after a controller +// restart and still executes the lease-aware workflow. +func (s *RPCService) SuspendActorForReconciler(ctx context.Context, req *ateapipb.SuspendActorRequest) (*ateapipb.SuspendActorResponse, error) { + actorRef := resources.ActorRefFromObjectRef(req.GetActor()) + actor, err := s.actorWorkflow.SuspendActorForReconciler(ctx, actorRef) + if err != nil { + return nil, err + } + return &ateapipb.SuspendActorResponse{Actor: actor}, nil +} + +func validateSuspendActorWithLeaseRequest(ctx context.Context, req *ateapipb.SuspendActorWithLeaseRequest) field.ErrorList { + op := operation.Operation{Type: operation.Create} + return Validate_SuspendActorWithLeaseRequest(ctx, op, nil, req, nil) +} + +func (s *RPCService) RenewActorLease(ctx context.Context, req *ateapipb.RenewActorLeaseRequest) (*ateapipb.RenewActorLeaseResponse, error) { + if errs := validateRenewActorLeaseRequest(ctx, req); len(errs) > 0 { + return nil, toGRPCStatusError(errs) + } + actorRef := resources.ActorRefFromObjectRef(req.GetActor()) + setSpanActorRefAttributes(ctx, actorRef) + + actor, err := s.impl.GetActor(ctx, actorRef) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil, status.Errorf(codes.NotFound, "Actor %s not found", actorRef) + } + return nil, err + } + if actor.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { + return nil, status.Error(codes.FailedPrecondition, "actor is not running") + } + renewed, err := s.impl.RenewActorRuntimeLease(ctx, actor.GetMetadata().GetUid(), req.GetLease().GetToken(), req.GetLease().GetGeneration()) + if errors.Is(err, store.ErrRuntimeLeaseInvalid) { + return nil, status.Error(codes.FailedPrecondition, "actor runtime lease is no longer current") + } + if err != nil { + return nil, err + } + return &ateapipb.RenewActorLeaseResponse{Lease: actorRuntimeLeaseProto(renewed)}, nil +} + +func validateRenewActorLeaseRequest(ctx context.Context, req *ateapipb.RenewActorLeaseRequest) field.ErrorList { + op := operation.Operation{Type: operation.Create} + return Validate_RenewActorLeaseRequest(ctx, op, nil, req, nil) +} + func validateSuspendActorRequest(ctx context.Context, req *ateapipb.SuspendActorRequest) field.ErrorList { // Call the generated validation. op := operation.Operation{Type: operation.Create} diff --git a/cmd/ateapi/internal/controlapi/runtime_lease.go b/cmd/ateapi/internal/controlapi/runtime_lease.go new file mode 100644 index 0000000000..3d50685dd8 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/runtime_lease.go @@ -0,0 +1,38 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func actorRuntimeLeaseProto(lease *store.ActorRuntimeLease) *ateapipb.ActorLease { + if lease == nil { + return nil + } + return &ateapipb.ActorLease{ + Token: lease.Token, + Generation: lease.Generation, + ExpiresAt: timestamppb.New(lease.ExpiresAt), + } +} + +func runtimeLeaseMatchesProto(lease *store.ActorRuntimeLease, requested *ateapipb.ActorLease) bool { + return requested != nil && lease != nil && + lease.Token == requested.GetToken() && + lease.Generation == requested.GetGeneration() +} diff --git a/cmd/ateapi/internal/controlapi/runtime_lease_reconciler.go b/cmd/ateapi/internal/controlapi/runtime_lease_reconciler.go new file mode 100644 index 0000000000..21cdf3c71c --- /dev/null +++ b/cmd/ateapi/internal/controlapi/runtime_lease_reconciler.go @@ -0,0 +1,195 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/internal/ateattr" + "github.com/agent-substrate/substrate/internal/proto/ateletpb" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +const ( + runtimeLeaseHeartbeatInterval = 30 * time.Second + runtimeLeaseReconcileLimit = 100 +) + +type runtimeLeaseReconcilerStore interface { + actorWorkflowStore + ListExpiredActorRuntimeLeases(ctx context.Context, limit int) ([]store.ActorRuntimeLease, error) +} + +type runtimeLeaseReconciler struct { + store runtimeLeaseReconcilerStore + terminate func(context.Context, *ateapipb.Actor) error +} + +func newRuntimeLeaseReconciler(st runtimeLeaseReconcilerStore, terminate func(context.Context, *ateapipb.Actor) error) *runtimeLeaseReconciler { + return &runtimeLeaseReconciler{store: st, terminate: terminate} +} + +func (r *runtimeLeaseReconciler) Start(ctx context.Context) { + go func() { + r.reconcile(ctx) + ticker := time.NewTicker(runtimeLeaseHeartbeatInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + r.reconcile(ctx) + } + } + }() +} + +func (r *runtimeLeaseReconciler) reconcile(ctx context.Context) { + leases, err := r.store.ListExpiredActorRuntimeLeases(ctx, runtimeLeaseReconcileLimit) + if err != nil { + slog.WarnContext(ctx, "failed to list expired actor runtime leases", "error", err) + return + } + for _, lease := range leases { + if err := r.reconcileOne(ctx, lease); err != nil { + slog.WarnContext(ctx, "failed to reclaim expired actor runtime lease", + "actor_uid", lease.ActorUID, "actor", lease.ActorRef, "error", err) + } + } +} + +func (r *runtimeLeaseReconciler) reconcileOne(ctx context.Context, expired store.ActorRuntimeLease) error { + leaseCtx, operationLease, err := acquireLease(ctx, r.store, "lease:actor:"+expired.ActorRef.Atespace+":"+expired.ActorRef.Name, "actor") + if err != nil { + return err + } + defer operationLease.Close() + + current, err := r.store.GetActorRuntimeLease(leaseCtx, expired.ActorUID) + if errors.Is(err, store.ErrNotFound) { + return nil + } + if err != nil { + return err + } + if !runtimeLeaseEqual(current, &expired) || current.ExpiresAt.After(time.Now()) { + return nil + } + if err := r.store.ClaimExpiredActorRuntimeLease(leaseCtx, expired.ActorUID, expired.Token, expired.Generation); err != nil { + if errors.Is(err, store.ErrRuntimeLeaseInvalid) { + return nil + } + return err + } + + actor, err := r.store.GetActor(leaseCtx, expired.ActorRef) + if errors.Is(err, store.ErrNotFound) { + return r.store.DeleteActorRuntimeLease(leaseCtx, expired.ActorUID, expired.Token, expired.Generation) + } + if err != nil { + return err + } + if actor.GetMetadata().GetUid() != expired.ActorUID { + // The name now addresses a replacement actor. The old UID's lease can + // be removed, but the replacement is never touched by this reclaim. + return r.store.DeleteActorRuntimeLease(leaseCtx, expired.ActorUID, expired.Token, expired.Generation) + } + + state := actor.GetStatus().GetState() + terminalWithoutWorkload := state == ateapipb.ActorState_ACTOR_STATE_SUSPENDED || + state == ateapipb.ActorState_ACTOR_STATE_PAUSED || + state == ateapipb.ActorState_ACTOR_STATE_CRASHED + if actor.GetStatus().GetWorkerAssignment() != nil { + if err := r.terminate(leaseCtx, actor); err != nil { + return err + } + } + if !terminalWithoutWorkload || actor.GetStatus().GetWorkerAssignment() != nil { + // crashActor releases the worker before publishing CRASHED. It does + // not checkpoint or alter the actor's durable snapshot fields. + if err := crashActor(leaseCtx, r.store, expired.ActorRef, ateattr.OperationResume, ateattr.ReasonUnknown); err != nil { + return err + } + } + + // Re-check the exact row after termination/release. The reclaim claim fences + // renewal; this keeps deletion fail-closed if a future caller changes that + // ordering. + current, err = r.store.GetActorRuntimeLease(leaseCtx, expired.ActorUID) + if errors.Is(err, store.ErrNotFound) { + return nil + } + if err != nil { + return err + } + if !runtimeLeaseEqual(current, &expired) { + return nil + } + return r.store.DeleteActorRuntimeLease(leaseCtx, expired.ActorUID, expired.Token, expired.Generation) +} + +func runtimeLeaseEqual(a, b *store.ActorRuntimeLease) bool { + return a != nil && b != nil && a.ActorUID == b.ActorUID && + a.ActorRef == b.ActorRef && a.Token == b.Token && a.Generation == b.Generation +} + +func (w *ActorWorkflow) terminateActorWorkload(ctx context.Context, actor *ateapipb.Actor) error { + if w.terminateWorkload != nil { + return w.terminateWorkload(ctx, actor) + } + return w.terminateActorWorkloadNative(ctx, actor) +} + +func (w *ActorWorkflow) terminateActorWorkloadNative(ctx context.Context, actor *ateapipb.Actor) error { + assignment := actor.GetStatus().GetWorkerAssignment() + if assignment == nil { + return nil + } + actorTemplate, err := resolveActorTemplate(ctx, w.store, actor) + if err != nil { + return fmt.Errorf("while resolving actor template for termination: %w", err) + } + spec, err := workloadSpecFromActorTemplate(actorTemplate, actor) + if err != nil { + return fmt.Errorf("while resolving workload for termination: %w", err) + } + conn, err := w.dialer.DialForWorker(assignment.GetWorkerNamespace(), assignment.GetWorkerPod()) + if errors.Is(err, ErrWorkerPodNotFound) { + // The workload is already gone with its worker pod. + return nil + } + if err != nil { + return fmt.Errorf("while getting atelet conn for expired actor: %w", err) + } + _, err = ateletpb.NewAteomHerderClient(conn).Terminate(ctx, &ateletpb.TerminateRequest{ + TargetAteomUid: assignment.GetWorkerPodUid(), + Atespace: actor.GetMetadata().GetAtespace(), + ActorName: actor.GetMetadata().GetName(), + ActorUid: actor.GetMetadata().GetUid(), + ActorTemplateAtespace: actor.GetActorTemplate().GetAtespace(), + ActorTemplateName: actor.GetActorTemplate().GetName(), + Spec: spec, + }) + if err != nil { + return fmt.Errorf("while terminating expired actor workload: %w", err) + } + return nil +} diff --git a/cmd/ateapi/internal/controlapi/runtime_lease_test.go b/cmd/ateapi/internal/controlapi/runtime_lease_test.go new file mode 100644 index 0000000000..a907c045ba --- /dev/null +++ b/cmd/ateapi/internal/controlapi/runtime_lease_test.go @@ -0,0 +1,202 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +func TestSuspendActorWithLease_StaleTupleFailsClosed(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-a"}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}, + }) + issued, err := persistence.IssueActorRuntimeLease(ctx, actor.GetMetadata().GetUid(), resources.ActorRefFromActor(actor)) + if err != nil { + t.Fatalf("IssueActorRuntimeLease: %v", err) + } + w := &ActorWorkflow{store: persistence} + _, err = w.SuspendActorWithLease(ctx, resources.ActorRefFromActor(actor), &ateapipb.ActorLease{ + Token: issued.Token, + Generation: issued.Generation + 1, + }) + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("SuspendActorWithLease error = %v, want FailedPrecondition", err) + } + stored, err := persistence.GetActor(ctx, resources.ActorRefFromActor(actor)) + if err != nil { + t.Fatalf("GetActor: %v", err) + } + if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_RUNNING { + t.Errorf("actor state after stale suspend = %v, want RUNNING", stored.GetStatus().GetState()) + } + if _, err := persistence.GetActorRuntimeLease(ctx, actor.GetMetadata().GetUid()); err != nil { + t.Fatalf("runtime lease after stale suspend: %v", err) + } +} + +type idempotentRuntimeLeaseStore struct { + store.Interface + actor *ateapipb.Actor + runtime *store.ActorRuntimeLease + deleteCalls int +} + +func (s *idempotentRuntimeLeaseStore) GetActor(_ context.Context, _ resources.ActorRef) (*ateapipb.Actor, error) { + return proto.Clone(s.actor).(*ateapipb.Actor), nil +} + +func (s *idempotentRuntimeLeaseStore) GetActorRuntimeLease(_ context.Context, _ string) (*store.ActorRuntimeLease, error) { + if s.runtime == nil { + return nil, store.ErrNotFound + } + lease := *s.runtime + return &lease, nil +} + +func (s *idempotentRuntimeLeaseStore) DeleteActorRuntimeLease(_ context.Context, actorUID, token string, generation int64) error { + s.deleteCalls++ + if s.runtime == nil || s.runtime.ActorUID != actorUID || s.runtime.Token != token || s.runtime.Generation != generation { + return store.ErrRuntimeLeaseInvalid + } + s.runtime = nil + return nil +} + +func (s *idempotentRuntimeLeaseStore) ClaimExpiredActorRuntimeLease(_ context.Context, actorUID, token string, generation int64) error { + if s.runtime == nil || s.runtime.ActorUID != actorUID || s.runtime.Token != token || s.runtime.Generation != generation { + return store.ErrRuntimeLeaseInvalid + } + return nil +} + +func TestRuntimeLeaseReconciler_IdempotentAndDoesNotMutateSnapshot(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + actor := &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-a", Uid: "actor-uid"}, + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_CRASHED, + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/last"}, + InProgressSnapshotName: "in-flight", + CurrentActorTemplateUid: "template-uid", + }, + } + ref := resources.ActorRefFromActor(actor) + lease := &store.ActorRuntimeLease{ + ActorUID: actor.GetMetadata().GetUid(), + ActorRef: ref, + Token: "expired-token", + Generation: 7, + ExpiresAt: time.Now().Add(-time.Minute), + } + fake := &idempotentRuntimeLeaseStore{Interface: persistence, actor: actor, runtime: lease} + terminateCalls := 0 + r := newRuntimeLeaseReconciler(fake, func(context.Context, *ateapipb.Actor) error { + terminateCalls++ + return errors.New("must not terminate an already crashed actor") + }) + if err := r.reconcileOne(ctx, *lease); err != nil { + t.Fatalf("first reconcileOne: %v", err) + } + if err := r.reconcileOne(ctx, *lease); err != nil { + t.Fatalf("second reconcileOne: %v", err) + } + if terminateCalls != 0 { + t.Errorf("terminate calls = %d, want 0", terminateCalls) + } + if fake.deleteCalls != 1 { + t.Errorf("delete calls = %d, want 1", fake.deleteCalls) + } + if got := fake.actor.GetStatus().GetExternalSnapshot().GetSnapshotUri(); got != "gs://bucket/last" { + t.Errorf("durable snapshot = %q, want last snapshot", got) + } + if got := fake.actor.GetStatus().GetInProgressSnapshotName(); got != "in-flight" { + t.Errorf("in-progress snapshot = %q, want unchanged", got) + } +} + +func TestCleanupFailedResume_TerminatesBeforeReleaseAndClearsLease(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + workerName := testWorkerUID("failed-resume-pod") + if _, err := persistence.CreateWorker(ctx, &ateapipb.Worker{ + Metadata: &ateapipb.ResourceMetadata{Name: workerName}, + WorkerNamespace: "worker-ns", + WorkerPool: "pool", + WorkerPod: "failed-resume-pod", + WorkerPodUid: workerName, + Status: &ateapipb.WorkerStatus{}, + }); err != nil { + t.Fatalf("CreateWorker: %v", err) + } + actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "actor-a"}, + Status: &ateapipb.ActorStatus{ + State: ateapipb.ActorState_ACTOR_STATE_RESUMING, + WorkerAssignment: &ateapipb.WorkerAssignment{Worker: &ateapipb.ObjectRef{Name: workerName}, WorkerNamespace: "worker-ns", WorkerPod: "failed-resume-pod", WorkerPodUid: workerName}, + ExternalSnapshot: &ateapipb.ExternalSnapshot{SnapshotUri: "gs://bucket/last"}, + InProgressSnapshotName: "in-flight", + }, + }) + seedAssignment(t, persistence, workerName, &ateapipb.ActorAssignment{Actor: &ateapipb.ObjectRef{Atespace: "team-a", Name: "actor-a"}, ActorUid: actor.GetMetadata().GetUid()}) + issued, err := persistence.IssueActorRuntimeLease(ctx, actor.GetMetadata().GetUid(), resources.ActorRefFromActor(actor)) + if err != nil { + t.Fatalf("IssueActorRuntimeLease: %v", err) + } + terminateCalls := 0 + w := &ActorWorkflow{ + store: persistence, + terminateWorkload: func(context.Context, *ateapipb.Actor) error { + terminateCalls++ + return nil + }, + } + if err := w.cleanupFailedResume(ctx, resources.ActorRefFromActor(actor), actor, actorRuntimeLeaseProto(issued)); err != nil { + t.Fatalf("cleanupFailedResume: %v", err) + } + if terminateCalls != 1 { + t.Errorf("terminate calls = %d, want 1", terminateCalls) + } + stored, err := persistence.GetActor(ctx, resources.ActorRefFromActor(actor)) + if err != nil { + t.Fatalf("GetActor: %v", err) + } + if stored.GetStatus().GetState() != ateapipb.ActorState_ACTOR_STATE_CRASHED || stored.GetStatus().GetWorkerAssignment() != nil { + t.Errorf("actor after cleanup = %v assignment %v, want CRASHED with no assignment", stored.GetStatus().GetState(), stored.GetStatus().GetWorkerAssignment()) + } + if stored.GetStatus().GetExternalSnapshot().GetSnapshotUri() != "gs://bucket/last" || stored.GetStatus().GetInProgressSnapshotName() != "in-flight" { + t.Errorf("snapshot state changed during cleanup: %v", stored.GetStatus()) + } + if _, err := persistence.GetActorRuntimeLease(ctx, actor.GetMetadata().GetUid()); !errors.Is(err, store.ErrNotFound) { + t.Errorf("runtime lease after cleanup = %v, want not found", err) + } + if assignment := firstAssignment(t, persistence, workerName); assignment != nil { + t.Errorf("worker assignment after cleanup = %v, want nil", assignment) + } +} diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index 2460b4638d..20d41cad4e 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -94,6 +94,12 @@ func NewRPCService( return s } +// StartRuntimeLeaseReconciler starts the native expiry loop. Multiple ate-api +// replicas safely race through the actor operation lease. +func (s *RPCService) StartRuntimeLeaseReconciler(ctx context.Context) { + newRuntimeLeaseReconciler(s.actorWorkflow.store, s.actorWorkflow.terminateActorWorkload).Start(ctx) +} + // serviceStore enumerates the exact storage methods needed by // the control API and nothing more. type serviceStore interface { @@ -123,6 +129,11 @@ type serviceStore interface { CreateWorker(ctx context.Context, worker *ateapipb.Worker) (*ateapipb.Worker, error) UpdateWorker(ctx context.Context, name string, precondition store.Precondition, mutate func(toUpdate *ateapipb.Worker) error) (*ateapipb.Worker, error) AcquireLease(ctx context.Context, key string) (*store.Lease, error) + IssueActorRuntimeLease(ctx context.Context, actorUID string, actorRef resources.ActorRef) (*store.ActorRuntimeLease, error) + GetActorRuntimeLease(ctx context.Context, actorUID string) (*store.ActorRuntimeLease, error) + RenewActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) (*store.ActorRuntimeLease, error) + DeleteActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) error + ListExpiredActorRuntimeLeases(ctx context.Context, limit int) ([]store.ActorRuntimeLease, error) } // GetPlugin retrieves a CSI volume plugin by driver name, dynamically discovering it if not present. @@ -177,3 +188,27 @@ func newServiceImpl( func (s *ServiceImpl) AcquireLease(ctx context.Context, key string) (*store.Lease, error) { return s.store.AcquireLease(ctx, key) } + +func (s *ServiceImpl) IssueActorRuntimeLease(ctx context.Context, actorUID string, actorRef resources.ActorRef) (*store.ActorRuntimeLease, error) { + return s.store.IssueActorRuntimeLease(ctx, actorUID, actorRef) +} + +func (s *ServiceImpl) GetActorRuntimeLease(ctx context.Context, actorUID string) (*store.ActorRuntimeLease, error) { + return s.store.GetActorRuntimeLease(ctx, actorUID) +} + +func (s *ServiceImpl) RenewActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) (*store.ActorRuntimeLease, error) { + return s.store.RenewActorRuntimeLease(ctx, actorUID, token, generation) +} + +func (s *ServiceImpl) ClaimExpiredActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) error { + return s.store.ClaimExpiredActorRuntimeLease(ctx, actorUID, token, generation) +} + +func (s *ServiceImpl) DeleteActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) error { + return s.store.DeleteActorRuntimeLease(ctx, actorUID, token, generation) +} + +func (s *ServiceImpl) ListExpiredActorRuntimeLeases(ctx context.Context, limit int) ([]store.ActorRuntimeLease, error) { + return s.store.ListExpiredActorRuntimeLeases(ctx, limit) +} diff --git a/cmd/ateapi/internal/controlapi/template_reconciler.go b/cmd/ateapi/internal/controlapi/template_reconciler.go index 47e65ee529..8af73845c5 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler.go @@ -69,6 +69,9 @@ type goldenActorControl interface { GetActor(ctx context.Context, req *ateapipb.GetActorRequest) (*ateapipb.Actor, error) ResumeActor(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) SuspendActor(ctx context.Context, req *ateapipb.SuspendActorRequest) (*ateapipb.SuspendActorResponse, error) + ResumeActorForReconciler(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) + SuspendActorForReconciler(ctx context.Context, req *ateapipb.SuspendActorRequest) (*ateapipb.SuspendActorResponse, error) + SuspendActorWithLease(ctx context.Context, req *ateapipb.SuspendActorWithLeaseRequest) (*ateapipb.SuspendActorResponse, error) } // ActorTemplateReconciler drives stored ActorTemplates through the golden @@ -194,6 +197,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource // Each iteration observes the golden actor, takes the one action that // fact demands, and re-observes; the pass ends at a terminal condition, // a deadline wait, or an error the workqueue retries. + var runtimeLease *ateapipb.ActorLease for { goldenSnapshotStatus := tmpl.GetStatus().GetGoldenSnapshotStatus() if goldenSnapshotStatus.GetErrorMessage() != "" { @@ -238,7 +242,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource return rem, nil } // Warmup done: suspend the golden actor and record its snapshot. - snapshot, err := r.suspendActor(ctx, goldenActorRef) + snapshot, err := r.suspendActor(ctx, goldenActorRef, runtimeLease) if err != nil { return 0, err } @@ -246,7 +250,7 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource case ateapipb.ActorState_ACTOR_STATE_SUSPENDING: // A previous pass died mid-suspend; retry suspend. - snapshot, err := r.suspendActor(ctx, goldenActorRef) + snapshot, err := r.suspendActor(ctx, goldenActorRef, runtimeLease) if err != nil { return 0, err } @@ -262,10 +266,12 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource // without being recorded. return 0, r.saveGoldenSnapshot(ctx, tmpl, actor.GetStatus().GetExternalSnapshot()) } - if _, err := r.control.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: goldenActorRef}); err != nil { + resumeResp, err := r.control.ResumeActorForReconciler(ctx, &ateapipb.ResumeActorRequest{Actor: goldenActorRef}) + if err != nil { // A crash during resume is observed as CRASHED on the retry. return 0, fmt.Errorf("while resuming golden actor: %w", err) } + runtimeLease = resumeResp.GetLease() deadline := time.Now().Add(goldenSnapshotWarmupFor(tmpl.GetContainers())) if tmpl, err = r.checkpoint(ctx, tmpl, func(snapshotStatus *ateapipb.GoldenSnapshotStatus) { snapshotStatus.TakeGoldenSnapshotAt = timestamppb.New(deadline) @@ -286,8 +292,14 @@ func (r *ActorTemplateReconciler) reconcileOne(ctx context.Context, ref resource // suspendActor suspends the golden actor and returns the external snapshot it // wrote. Reentrant: SuspendActor completes an in-flight suspend and is a no-op // on an already-suspended actor, returning the existing snapshot either way. -func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *ateapipb.ObjectRef) (*ateapipb.ExternalSnapshot, error) { - resp, err := r.control.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: goldenRef}) +func (r *ActorTemplateReconciler) suspendActor(ctx context.Context, goldenRef *ateapipb.ObjectRef, runtimeLease *ateapipb.ActorLease) (*ateapipb.ExternalSnapshot, error) { + var resp *ateapipb.SuspendActorResponse + var err error + if runtimeLease != nil { + resp, err = r.control.SuspendActorWithLease(ctx, &ateapipb.SuspendActorWithLeaseRequest{Actor: goldenRef, Lease: runtimeLease}) + } else { + resp, err = r.control.SuspendActorForReconciler(ctx, &ateapipb.SuspendActorRequest{Actor: goldenRef}) + } if err != nil { // A crash during suspend is observed as CRASHED on the retry. return nil, fmt.Errorf("while suspending golden actor: %w", err) diff --git a/cmd/ateapi/internal/controlapi/template_reconciler_test.go b/cmd/ateapi/internal/controlapi/template_reconciler_test.go index aac5806dcd..dbae857498 100644 --- a/cmd/ateapi/internal/controlapi/template_reconciler_test.go +++ b/cmd/ateapi/internal/controlapi/template_reconciler_test.go @@ -216,7 +216,11 @@ func (c *fakeGoldenControl) ResumeActor(_ context.Context, req *ateapipb.ResumeA return nil, c.resumeErr } c.goldenState = ateapipb.ActorState_ACTOR_STATE_RUNNING - return &ateapipb.ResumeActorResponse{}, nil + return &ateapipb.ResumeActorResponse{Lease: &ateapipb.ActorLease{Token: "golden-token", Generation: 1}}, nil +} + +func (c *fakeGoldenControl) ResumeActorForReconciler(ctx context.Context, req *ateapipb.ResumeActorRequest) (*ateapipb.ResumeActorResponse, error) { + return c.ResumeActor(ctx, req) } func (c *fakeGoldenControl) SuspendActor(_ context.Context, req *ateapipb.SuspendActorRequest) (*ateapipb.SuspendActorResponse, error) { @@ -235,6 +239,14 @@ func (c *fakeGoldenControl) SuspendActor(_ context.Context, req *ateapipb.Suspen }, nil } +func (c *fakeGoldenControl) SuspendActorForReconciler(ctx context.Context, req *ateapipb.SuspendActorRequest) (*ateapipb.SuspendActorResponse, error) { + return c.SuspendActor(ctx, req) +} + +func (c *fakeGoldenControl) SuspendActorWithLease(ctx context.Context, req *ateapipb.SuspendActorWithLeaseRequest) (*ateapipb.SuspendActorResponse, error) { + return c.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: req.GetActor()}) +} + func (c *fakeGoldenControl) callCounts() (creates, resumes, suspends int) { c.mu.Lock() defer c.mu.Unlock() diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index 7943aaf368..6a7b0f8e95 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -79,6 +79,7 @@ type ActorWorkflow struct { egressGatewayAddress string pluginRegistry VolumePluginRegistry objectStore objectstore.Store + terminateWorkload func(context.Context, *ateapipb.Actor) error } // NewActorWorkflow creates a new ActorWorkflow. instruments may be nil. @@ -131,6 +132,12 @@ type actorWorkflowStore interface { UpdateTag(ctx context.Context, tagRef resources.TagRef, precondition store.Precondition, mutate func(toUpdate *ateapipb.Tag) error) (*ateapipb.Tag, error) GetActorTemplate(ctx context.Context, templateRef resources.ActorTemplateRef) (*ateapipb.ActorTemplate, error) AcquireLease(ctx context.Context, key string) (*store.Lease, error) + IssueActorRuntimeLease(ctx context.Context, actorUID string, actorRef resources.ActorRef) (*store.ActorRuntimeLease, error) + GetActorRuntimeLease(ctx context.Context, actorUID string) (*store.ActorRuntimeLease, error) + RenewActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) (*store.ActorRuntimeLease, error) + ClaimExpiredActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) error + DeleteActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) error + ListExpiredActorRuntimeLeases(ctx context.Context, limit int) ([]store.ActorRuntimeLease, error) } // WorkerWorkflow handles the multi-step operations on a Worker. diff --git a/cmd/ateapi/internal/controlapi/workflow_resume.go b/cmd/ateapi/internal/controlapi/workflow_resume.go index c3b6fca08d..255b1f2038 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume.go @@ -72,10 +72,10 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto var tele restoreTelemetry var wasRunning bool - // Recorded before the lease so lease contention still counts as an attempt. - // Clean already-running no-ops are skipped: the router resumes per routed - // request, and recording those would sample at router QPS and bury - // cold-resume latency. + // The router calls this method for every request. Keep its established + // read-only running fast path; executor-owned callers use + // ResumeActorWithLease instead so an actor cannot be revived without an + // owner lease. defer func() { if err == nil && wasRunning { return @@ -84,10 +84,6 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto lifecycleOpAttrs(actor, actorTemplate, tele.SnapshotKind, tele.WireSnapshotScope)...) }() - // Routed requests call ResumeActor even when the actor is already running. - // Read before taking the distributed lease so that hot-path checks do not - // upsert and delete a PostgreSQL lease row. Any state that needs work is read - // again under the lease below. actor, err = w.store.GetActor(ctx, actorRef) if err != nil { return nil, false, err @@ -110,6 +106,9 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto if wasRunning = actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_RUNNING; wasRunning { return actor, false, nil } + if err = w.ensureNoRuntimeLease(leaseCtx, actor); err != nil { + return nil, false, err + } var created *ateapipb.Actor if created, err = w.ensureVolumesCreated(leaseCtx, actorRef, actor, actorTemplate); err != nil { return nil, false, err @@ -131,8 +130,188 @@ func (w *ActorWorkflow) ResumeActor(ctx context.Context, actorRef resources.Acto if running, err = w.finalizeRunning(leaseCtx, actorRef, actorTemplate); err != nil { return nil, false, err } + return running, true, nil +} + +func (w *ActorWorkflow) ensureNoRuntimeLease(ctx context.Context, actor *ateapipb.Actor) error { + _, err := w.store.GetActorRuntimeLease(ctx, actor.GetMetadata().GetUid()) + if err == nil { + return status.Error(codes.FailedPrecondition, "actor is owned by an executor runtime lease") + } + if errors.Is(err, store.ErrNotFound) { + return nil + } + return err +} + +// ResumeActorForReconciler is the trusted in-process path used by the golden +// actor controller. It reattaches to a persisted runtime lease before calling +// the same lease-checked workflow, so a controller restart cannot strand the +// golden workload behind an otherwise correctly fail-closed public API. +func (w *ActorWorkflow) ResumeActorForReconciler(ctx context.Context, actorRef resources.ActorRef, boot bool) (*ateapipb.Actor, bool, *ateapipb.ActorLease, error) { + actor, err := w.store.GetActor(ctx, actorRef) + if err != nil { + return nil, false, nil, err + } + stored, getErr := w.store.GetActorRuntimeLease(ctx, actor.GetMetadata().GetUid()) + if getErr != nil && !errors.Is(getErr, store.ErrNotFound) { + return nil, false, nil, getErr + } + requested := actorRuntimeLeaseProto(stored) + return w.ResumeActorWithLease(ctx, actorRef, boot, requested) +} + +// ResumeActorWithLease resumes an actor and returns the runtime lease that +// owns the resulting workload. A running actor may be reattached only with +// its matching lease; a legacy running actor without a lease row is assigned +// one while the actor operation lease is held. +func (w *ActorWorkflow) ResumeActorWithLease(ctx context.Context, actorRef resources.ActorRef, boot bool, requestedLease *ateapipb.ActorLease) (_ *ateapipb.Actor, resumed bool, runtimeLease *ateapipb.ActorLease, err error) { + start := time.Now() + var actor *ateapipb.Actor + var actorTemplate *ateapipb.ActorTemplate + var tele restoreTelemetry + var wasRunning bool + + // Recorded before the lease so lease contention still counts as an attempt. + // Clean already-running no-ops are skipped: the router resumes per routed + // request, and recording those would sample at router QPS and bury + // cold-resume latency. + defer func() { + if err == nil && wasRunning { + return + } + w.instruments.recordLifecycleOp(ctx, ateattr.OperationResume, start, err, + lifecycleOpAttrs(actor, actorTemplate, tele.SnapshotKind, tele.WireSnapshotScope)...) + }() + + leaseCtx, lease, err := w.acquireActorLease(ctx, actorRef) + if err != nil { + return nil, false, nil, err + } + defer lease.Close() + + var src resumeSnapshotSource + actor, actorTemplate, src, err = w.loadActorForResume(leaseCtx, actorRef, boot) + if err != nil { + return nil, false, nil, err + } + if wasRunning = actor.GetStatus().GetState() == ateapipb.ActorState_ACTOR_STATE_RUNNING; wasRunning { + runtimeLease, err = w.ensureRunningRuntimeLease(leaseCtx, actor, requestedLease) + return actor, false, runtimeLease, err + } + if requestedLease == nil { + // A fresh executor may claim only an unowned incarnation. A persisted + // lease belongs to the previous owner until the native reaper removes it; + // adopting it without its token would defeat hard-death fencing. + if err = w.ensureNoRuntimeLease(leaseCtx, actor); err != nil { + return nil, false, nil, err + } + } + var created *ateapipb.Actor + if created, err = w.ensureVolumesCreated(leaseCtx, actorRef, actor, actorTemplate); err != nil { + return nil, false, nil, err + } + actor = created + var worker *ateapipb.Worker + var assigned *ateapipb.Actor + if assigned, worker, err = w.ensureWorkerAssigned(leaseCtx, actorRef, actor, actorTemplate); err != nil { + return nil, false, nil, err + } + actor = assigned + if err = w.ensureVolumesAttached(leaseCtx, actor, worker, actorTemplate); err != nil { + return nil, false, nil, err + } + if tele, err = w.ensureAteletRestored(leaseCtx, actorRef, actor, actorTemplate, src); err != nil { + return nil, false, nil, err + } + runtimeLease, err = w.ensureRuntimeLease(leaseCtx, actor, requestedLease) + if err != nil { + if cleanupErr := w.cleanupFailedResume(leaseCtx, actorRef, actor, nil); cleanupErr != nil { + return nil, false, nil, errors.Join(err, cleanupErr) + } + return nil, false, nil, err + } + var running *ateapipb.Actor + if running, err = w.finalizeRunning(leaseCtx, actorRef, actorTemplate); err != nil { + if cleanupErr := w.cleanupFailedResume(leaseCtx, actorRef, actor, runtimeLease); cleanupErr != nil { + return nil, false, nil, errors.Join(err, cleanupErr) + } + return nil, false, nil, err + } actor = running - return actor, true, nil + return actor, true, runtimeLease, nil +} + +func (w *ActorWorkflow) cleanupFailedResume(ctx context.Context, actorRef resources.ActorRef, actor *ateapipb.Actor, runtimeLease *ateapipb.ActorLease) error { + if err := w.terminateActorWorkload(ctx, actor); err != nil { + return fmt.Errorf("while cleaning up failed resume workload: %w", err) + } + if err := crashActor(ctx, w.store, actorRef, ateattr.OperationResume, ateattr.ReasonUnknown); err != nil { + return fmt.Errorf("while crashing failed resume actor: %w", err) + } + if runtimeLease == nil { + return nil + } + stored, err := w.store.GetActorRuntimeLease(ctx, actor.GetMetadata().GetUid()) + if errors.Is(err, store.ErrNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("while checking failed resume runtime lease: %w", err) + } + if !runtimeLeaseMatchesProto(stored, runtimeLease) { + return store.ErrRuntimeLeaseInvalid + } + if err := w.store.DeleteActorRuntimeLease(ctx, stored.ActorUID, stored.Token, stored.Generation); err != nil { + return fmt.Errorf("while clearing failed resume runtime lease: %w", err) + } + return nil +} + +func (w *ActorWorkflow) ensureRuntimeLease(ctx context.Context, actor *ateapipb.Actor, requested *ateapipb.ActorLease) (*ateapipb.ActorLease, error) { + stored, err := w.store.GetActorRuntimeLease(ctx, actor.GetMetadata().GetUid()) + if err == nil { + if requested == nil || stored.ExpiresAt.Before(time.Now()) || !runtimeLeaseMatchesProto(stored, requested) { + return nil, status.Error(codes.FailedPrecondition, "actor runtime lease is stale or does not match") + } + return actorRuntimeLeaseProto(stored), nil + } + if !errors.Is(err, store.ErrNotFound) { + return nil, err + } + if requested != nil { + return nil, status.Error(codes.FailedPrecondition, "actor runtime lease is no longer current") + } + issued, err := w.store.IssueActorRuntimeLease(ctx, actor.GetMetadata().GetUid(), resources.ActorRefFromActor(actor)) + if err != nil { + return nil, err + } + return actorRuntimeLeaseProto(issued), nil +} + +func (w *ActorWorkflow) ensureRunningRuntimeLease(ctx context.Context, actor *ateapipb.Actor, requested *ateapipb.ActorLease) (*ateapipb.ActorLease, error) { + stored, err := w.store.GetActorRuntimeLease(ctx, actor.GetMetadata().GetUid()) + if errors.Is(err, store.ErrNotFound) { + if requested != nil { + return nil, status.Error(codes.FailedPrecondition, "actor runtime lease is no longer current") + } + issued, issueErr := w.store.IssueActorRuntimeLease(ctx, actor.GetMetadata().GetUid(), resources.ActorRefFromActor(actor)) + if issueErr != nil { + return nil, issueErr + } + return actorRuntimeLeaseProto(issued), nil + } + if err != nil { + return nil, err + } + if stored.ExpiresAt.Before(time.Now()) || !runtimeLeaseMatchesProto(stored, requested) { + return nil, status.Error(codes.FailedPrecondition, "actor runtime lease is stale or does not match") + } + renewed, err := w.store.RenewActorRuntimeLease(ctx, stored.ActorUID, stored.Token, stored.Generation) + if err != nil { + return nil, status.Error(codes.FailedPrecondition, "actor runtime lease is no longer current") + } + return actorRuntimeLeaseProto(renewed), nil } // validateGoldenSnapshotScope rejects a golden snapshot that does not carry diff --git a/cmd/ateapi/internal/controlapi/workflow_resume_test.go b/cmd/ateapi/internal/controlapi/workflow_resume_test.go index f5797d61a3..820b995251 100644 --- a/cmd/ateapi/internal/controlapi/workflow_resume_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_resume_test.go @@ -68,7 +68,7 @@ func (s *leaseCountingStore) AcquireLease(ctx context.Context, key string) (*sto return s.Interface.AcquireLease(ctx, key) } -func TestResumeActor_RunningFastPathDoesNotAcquireLease(t *testing.T) { +func TestResumeActorWithLease_RunningLegacyActorGetsRuntimeLeaseUnderOperationLease(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ @@ -78,7 +78,7 @@ func TestResumeActor_RunningFastPathDoesNotAcquireLease(t *testing.T) { st := &leaseCountingStore{Interface: persistence} w := &ActorWorkflow{store: st} - got, resumed, err := w.ResumeActor(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, false) + got, resumed, runtimeLease, err := w.ResumeActorWithLease(ctx, resources.ActorRef{Atespace: "team-a", Name: "id1"}, false, nil) if err != nil { t.Fatalf("ResumeActor: %v", err) } @@ -88,9 +88,69 @@ func TestResumeActor_RunningFastPathDoesNotAcquireLease(t *testing.T) { if !proto.Equal(got, created) { t.Errorf("ResumeActor actor = %v, want %v", got, created) } + if st.acquireCalls != 1 { + t.Errorf("AcquireLease calls = %d, want 1", st.acquireCalls) + } + if runtimeLease == nil { + t.Fatal("runtime lease = nil, want lease") + } + lease, err := persistence.GetActorRuntimeLease(ctx, created.GetMetadata().GetUid()) + if err != nil { + t.Fatalf("GetActorRuntimeLease: %v", err) + } + if lease.Generation < 1 || lease.Token == "" { + t.Fatalf("runtime lease = %+v, want token and positive generation", lease) + } +} + +func TestResumeActor_RunningPathDoesNotClaimRuntimeLease(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "router-actor"}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}, + }) + st := &leaseCountingStore{Interface: persistence} + w := &ActorWorkflow{store: st} + + got, resumed, err := w.ResumeActor(ctx, resources.ActorRefFromActor(created), false) + if err != nil { + t.Fatalf("ResumeActor: %v", err) + } + if resumed { + t.Fatal("ResumeActor resumed = true, want false") + } + if !proto.Equal(got, created) { + t.Errorf("ResumeActor actor = %v, want %v", got, created) + } if st.acquireCalls != 0 { t.Errorf("AcquireLease calls = %d, want 0", st.acquireCalls) } + if _, err := persistence.GetActorRuntimeLease(ctx, created.GetMetadata().GetUid()); !errors.Is(err, store.ErrNotFound) { + t.Errorf("runtime lease after router path = %v, want not found", err) + } +} + +func TestResumeActorWithLease_RunningActorRequiresCurrentLease(t *testing.T) { + ctx := context.Background() + persistence := newTestPersistence(t) + created := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: "team-a", Name: "owned-actor"}, + Status: &ateapipb.ActorStatus{State: ateapipb.ActorState_ACTOR_STATE_RUNNING}, + }) + _, err := persistence.IssueActorRuntimeLease(ctx, created.GetMetadata().GetUid(), resources.ActorRefFromActor(created)) + if err != nil { + t.Fatalf("IssueActorRuntimeLease: %v", err) + } + w := &ActorWorkflow{store: persistence} + + _, _, _, err = w.ResumeActorWithLease(ctx, resources.ActorRefFromActor(created), false, nil) + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("ResumeActorWithLease error = %v, want FailedPrecondition", err) + } + if _, err := persistence.GetActorRuntimeLease(ctx, created.GetMetadata().GetUid()); err != nil { + t.Fatalf("runtime lease after rejected claim: %v", err) + } } // TestFinalizeRunning_RecordsSprintTemplate verifies committing RUNNING stamps diff --git a/cmd/ateapi/internal/controlapi/workflow_suspend.go b/cmd/ateapi/internal/controlapi/workflow_suspend.go index d76c9591ae..0305c6a4ac 100644 --- a/cmd/ateapi/internal/controlapi/workflow_suspend.go +++ b/cmd/ateapi/internal/controlapi/workflow_suspend.go @@ -39,6 +39,76 @@ import ( // the steps a previous attempt completed, deriving progress from the // persisted actor alone. func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { + leaseCtx, lease, err := w.acquireActorLease(ctx, actorRef) + if err != nil { + return nil, err + } + defer lease.Close() + actor, err := w.store.GetActor(leaseCtx, actorRef) + if err != nil { + return nil, err + } + if err := w.ensureNoRuntimeLease(leaseCtx, actor); err != nil { + return nil, err + } + return w.suspendActorHeld(leaseCtx, actorRef) +} + +// SuspendActorWithLease requires the runtime lease for the actor incarnation. +// The actor operation lease is acquired before validating the tuple, so a +// stale caller cannot reach any suspend step or a replacement actor. +func (w *ActorWorkflow) SuspendActorWithLease(ctx context.Context, actorRef resources.ActorRef, requested *ateapipb.ActorLease) (_ *ateapipb.Actor, err error) { + leaseCtx, lease, err := w.acquireActorLease(ctx, actorRef) + if err != nil { + return nil, err + } + defer lease.Close() + + actor, err := w.store.GetActor(leaseCtx, actorRef) + if err != nil { + return nil, err + } + runtimeLease, err := w.store.GetActorRuntimeLease(leaseCtx, actor.GetMetadata().GetUid()) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + return nil, status.Error(codes.FailedPrecondition, "actor runtime lease is no longer current") + } + return nil, err + } + if runtimeLease.ExpiresAt.Before(time.Now()) || !runtimeLeaseMatchesProto(runtimeLease, requested) { + return nil, status.Error(codes.FailedPrecondition, "actor runtime lease is stale or does not match") + } + + actor, err = w.suspendActorHeld(leaseCtx, actorRef) + if err != nil { + return nil, err + } + if err := w.store.DeleteActorRuntimeLease(leaseCtx, runtimeLease.ActorUID, runtimeLease.Token, runtimeLease.Generation); err != nil { + return nil, fmt.Errorf("while clearing suspended actor runtime lease: %w", err) + } + return actor, nil +} + +// SuspendActorForReconciler reattaches to the persisted runtime lease for a +// trusted in-process controller after a restart, then uses the same +// lease-aware suspend path. Actors from before runtime leases existed retain +// the compatibility path. +func (w *ActorWorkflow) SuspendActorForReconciler(ctx context.Context, actorRef resources.ActorRef) (*ateapipb.Actor, error) { + actor, err := w.store.GetActor(ctx, actorRef) + if err != nil { + return nil, err + } + stored, err := w.store.GetActorRuntimeLease(ctx, actor.GetMetadata().GetUid()) + if errors.Is(err, store.ErrNotFound) { + return w.SuspendActor(ctx, actorRef) + } + if err != nil { + return nil, err + } + return w.SuspendActorWithLease(ctx, actorRef, actorRuntimeLeaseProto(stored)) +} + +func (w *ActorWorkflow) suspendActorHeld(ctx context.Context, actorRef resources.ActorRef) (_ *ateapipb.Actor, err error) { start := time.Now() var actor *ateapipb.Actor var actorTemplate *ateapipb.ActorTemplate @@ -55,13 +125,7 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act w.instruments.recordLifecycleOp(ctx, ateattr.OperationSuspend, start, err, attrs...) }() - leaseCtx, lease, err := w.acquireActorLease(ctx, actorRef) - if err != nil { - return nil, err - } - defer lease.Close() - - actor, actorTemplate, err = w.loadActorForSuspend(leaseCtx, actorRef) + actor, actorTemplate, err = w.loadActorForSuspend(ctx, actorRef) if err != nil { return nil, err } @@ -76,26 +140,26 @@ func (w *ActorWorkflow) SuspendActor(ctx context.Context, actorRef resources.Act // alone can no longer tell the two origins apart. fromPaused := isPausedOriginSuspend(actor) var marked *ateapipb.Actor - if marked, err = w.ensureMarkedSuspending(leaseCtx, actorRef, actor, actorTemplate); err != nil { + if marked, err = w.ensureMarkedSuspending(ctx, actorRef, actor, actorTemplate); err != nil { return nil, err } actor = marked if fromPaused { - wireSnapshotScope, err = w.ensurePausedSnapshotUploaded(leaseCtx, actorRef, actor, actorTemplate) + wireSnapshotScope, err = w.ensurePausedSnapshotUploaded(ctx, actorRef, actor, actorTemplate) } else { - wireSnapshotScope, err = w.ensureAteletSuspended(leaseCtx, actorRef, actor, actorTemplate) + wireSnapshotScope, err = w.ensureAteletSuspended(ctx, actorRef, actor, actorTemplate) } if err != nil { return nil, err } - if err = w.ensureVolumesDetached(leaseCtx, actor, actorTemplate, "DetachVolumes", ateattr.OperationSuspend); err != nil { + if err = w.ensureVolumesDetached(ctx, actor, actorTemplate, "DetachVolumes", ateattr.OperationSuspend); err != nil { return nil, err } // FinalizeSuspended clears the WorkerAssignment the labels read, so snapshot // them here, as crash.go does for the crash counter. finalAttrs = lifecycleOpAttrs(actor, actorTemplate, "", wireSnapshotScope) var finalized *ateapipb.Actor - if finalized, err = w.ensureSuspendedFinalized(leaseCtx, actorRef, actorTemplate); err != nil { + if finalized, err = w.ensureSuspendedFinalized(ctx, actorRef, actorTemplate); err != nil { return nil, err } actor = finalized diff --git a/cmd/ateapi/internal/controlapi/zz_generated.validation.go b/cmd/ateapi/internal/controlapi/zz_generated.validation.go index 616c5c9af5..c7b09cc920 100644 --- a/cmd/ateapi/internal/controlapi/zz_generated.validation.go +++ b/cmd/ateapi/internal/controlapi/zz_generated.validation.go @@ -245,6 +245,77 @@ func Validate_Actor( return errs } +// Validate_ActorLease validates an instance of ActorLease according +// to declarative validation rules in the API schema. +func Validate_ActorLease( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.ActorLease) (errs field.ErrorList) { + + { // field ateapipb.ActorLease.Token + fn := func( + fldPath *field.Path, + obj, oldObj *string, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ActorLease) *string { + return &oldObj.Token + }) + errs = append(errs, fn(fldPath.Child("token"), &obj.Token, oldVal, oldObj != nil)...) + } + + { // field ateapipb.ActorLease.Generation + fn := func( + fldPath *field.Path, + obj, oldObj *int64, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + if e := validate.Minimum(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ActorLease) *int64 { + return &oldObj.Generation + }) + errs = append(errs, fn(fldPath.Child("generation"), &obj.Generation, oldVal, oldObj != nil)...) + } + + // field ateapipb.ActorLease.ExpiresAt has no validation + return errs +} + // Validate_ActorMetadataDataSource validates an instance of ActorMetadataDataSource according // to declarative validation rules in the API schema. func Validate_ActorMetadataDataSource( @@ -5071,6 +5142,92 @@ func Validate_PauseActorRequest( return errs } +// Validate_RenewActorLeaseRequest validates an instance of RenewActorLeaseRequest according +// to declarative validation rules in the API schema. +func Validate_RenewActorLeaseRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.RenewActorLeaseRequest) (errs field.ErrorList) { + + { // field ateapipb.RenewActorLeaseRequest.Actor + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ObjectRef, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + func() { // cohort = "atespace" + earlyReturn := false + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.RequiredValue).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + }() + // call the type's validation function + errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.RenewActorLeaseRequest) *ateapipb.ObjectRef { + return oldObj.Actor + }) + errs = append(errs, fn(fldPath.Child("actor"), obj.Actor, oldVal, oldObj != nil)...) + } + + { // field ateapipb.RenewActorLeaseRequest.Lease + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ActorLease, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_ActorLease(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.RenewActorLeaseRequest) *ateapipb.ActorLease { + return oldObj.Lease + }) + errs = append(errs, fn(fldPath.Child("lease"), obj.Lease, oldVal, oldObj != nil)...) + } + + return errs +} + // Validate_ResourceMetadata validates an instance of ResourceMetadata according // to declarative validation rules in the API schema. func Validate_ResourceMetadata( @@ -5428,6 +5585,64 @@ func Validate_ResumeActorRequest( errs = append(errs, fn(fldPath.Child("boot"), &obj.Boot, oldVal, oldObj != nil)...) } + { // field ateapipb.ResumeActorRequest.Lease + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ActorLease, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_ActorLease(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ResumeActorRequest) *ateapipb.ActorLease { + return oldObj.Lease + }) + errs = append(errs, fn(fldPath.Child("lease"), obj.Lease, oldVal, oldObj != nil)...) + } + + { // field ateapipb.ResumeActorRequest.ClaimRuntimeLease + fn := func( + fldPath *field.Path, + obj, oldObj *bool, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.OptionalValue(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.ResumeActorRequest) *bool { + return &oldObj.ClaimRuntimeLease + }) + errs = append(errs, fn(fldPath.Child("claim_runtime_lease"), &obj.ClaimRuntimeLease, oldVal, oldObj != nil)...) + } + return errs } @@ -5897,6 +6112,92 @@ func Validate_SuspendActorRequest( return errs } +// Validate_SuspendActorWithLeaseRequest validates an instance of SuspendActorWithLeaseRequest according +// to declarative validation rules in the API schema. +func Validate_SuspendActorWithLeaseRequest( + ctx context.Context, op operation.Operation, fldPath *field.Path, + obj, oldObj *ateapipb.SuspendActorWithLeaseRequest) (errs field.ErrorList) { + + { // field ateapipb.SuspendActorWithLeaseRequest.Actor + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ObjectRef, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + func() { // cohort = "atespace" + earlyReturn := false + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.RequiredValue).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.Subfield(ctx, op, fldPath, obj, oldObj, "atespace", + func(o *ateapipb.ObjectRef) *string { return &o.Atespace }, validate.DirectEqual, validate.OptionalValue).MarkShortCircuit(); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + }() + // call the type's validation function + errs = append(errs, Validate_ObjectRef(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SuspendActorWithLeaseRequest) *ateapipb.ObjectRef { + return oldObj.Actor + }) + errs = append(errs, fn(fldPath.Child("actor"), obj.Actor, oldVal, oldObj != nil)...) + } + + { // field ateapipb.SuspendActorWithLeaseRequest.Lease + fn := func( + fldPath *field.Path, + obj, oldObj *ateapipb.ActorLease, + oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update { + if ateDeepEqual(obj, oldObj) { + return nil + } + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredPointer(ctx, op, fldPath, obj, oldObj).MarkShortCircuit(); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_ActorLease(ctx, op, fldPath, obj, oldObj)...) + return + } + oldVal := safe.Field(oldObj, + func(oldObj *ateapipb.SuspendActorWithLeaseRequest) *ateapipb.ActorLease { + return oldObj.Lease + }) + errs = append(errs, fn(fldPath.Child("lease"), obj.Lease, oldVal, oldObj != nil)...) + } + + return errs +} + var unionMembershipFor_github_com_agent_substrate_substrate_pkg_proto_ateapipb_SystemInfoDataSource_ = validate.NewUnionMembership(validate.NewUnionMember("actor_metadata"), validate.NewUnionMember("trust_bundle")) // Validate_SystemInfoDataSource validates an instance of SystemInfoDataSource according diff --git a/cmd/ateapi/internal/store/atepg/atepg.go b/cmd/ateapi/internal/store/atepg/atepg.go index fabede86db..455b939f7c 100644 --- a/cmd/ateapi/internal/store/atepg/atepg.go +++ b/cmd/ateapi/internal/store/atepg/atepg.go @@ -1747,7 +1747,11 @@ func (p *Persistence) ListWorkers(ctx context.Context, opts store.ListOptions) ( // defaultLeaseTTL is how long a lease may go unrenewed before another client // can reclaim it. -const defaultLeaseTTL = 30 * time.Second +const ( + defaultLeaseTTL = 30 * time.Second + runtimeLeaseTTL = 120 * time.Second + defaultRuntimeLeaseListLimit = 100 +) func (p *Persistence) AcquireLease(ctx context.Context, key string) (*store.Lease, error) { ttl := p.leaseTTL @@ -1918,3 +1922,125 @@ func (p *Persistence) releaseLease(ctx context.Context, key, token string) error } return nil } + +// --- Actor runtime leases --- + +func (p *Persistence) IssueActorRuntimeLease(ctx context.Context, actorUID string, actorRef resources.ActorRef) (*store.ActorRuntimeLease, error) { + lease := &store.ActorRuntimeLease{} + err := p.pool.QueryRow(ctx, ` + INSERT INTO actor_runtime_leases + (actor_uid, actor_atespace, actor_name, token, generation, expires_at) + VALUES ($1, $2, $3, $4, nextval('actor_runtime_lease_generation_seq'), + clock_timestamp() + make_interval(secs => $5)) + ON CONFLICT (actor_uid) DO NOTHING + RETURNING actor_uid, token, generation, expires_at`, + actorUID, actorRef.Atespace, actorRef.Name, uuid.NewString(), runtimeLeaseTTL.Seconds()).Scan( + &lease.ActorUID, &lease.Token, &lease.Generation, &lease.ExpiresAt) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, store.ErrLeaseConflict + } + return nil, fmt.Errorf("issuing runtime lease for actor %q: %w", actorUID, err) + } + lease.ActorRef = actorRef + return lease, nil +} + +func (p *Persistence) GetActorRuntimeLease(ctx context.Context, actorUID string) (*store.ActorRuntimeLease, error) { + lease := &store.ActorRuntimeLease{ActorUID: actorUID} + err := p.pool.QueryRow(ctx, ` + SELECT actor_atespace, actor_name, token, generation, expires_at + FROM actor_runtime_leases + WHERE actor_uid = $1`, actorUID).Scan( + &lease.ActorRef.Atespace, &lease.ActorRef.Name, &lease.Token, &lease.Generation, &lease.ExpiresAt) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, store.ErrNotFound + } + return nil, fmt.Errorf("getting runtime lease for actor %q: %w", actorUID, err) + } + return lease, nil +} + +func (p *Persistence) RenewActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) (*store.ActorRuntimeLease, error) { + lease := &store.ActorRuntimeLease{ActorUID: actorUID, Token: token, Generation: generation} + err := p.pool.QueryRow(ctx, ` + UPDATE actor_runtime_leases + SET expires_at = clock_timestamp() + make_interval(secs => $4) + WHERE actor_uid = $1 + AND token = $2 + AND generation = $3 + AND reclaiming_until <= clock_timestamp() + AND expires_at > clock_timestamp() + RETURNING actor_atespace, actor_name, expires_at`, + actorUID, token, generation, runtimeLeaseTTL.Seconds()).Scan( + &lease.ActorRef.Atespace, &lease.ActorRef.Name, &lease.ExpiresAt) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, store.ErrRuntimeLeaseInvalid + } + return nil, fmt.Errorf("renewing runtime lease for actor %q: %w", actorUID, err) + } + return lease, nil +} + +func (p *Persistence) ClaimExpiredActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) error { + result, err := p.pool.Exec(ctx, ` + UPDATE actor_runtime_leases + SET reclaiming_until = clock_timestamp() + interval '5 minutes' + WHERE actor_uid = $1 + AND token = $2 + AND generation = $3 + AND expires_at <= clock_timestamp() + AND reclaiming_until <= clock_timestamp()`, actorUID, token, generation) + if err != nil { + return fmt.Errorf("claiming expired runtime lease for actor %q: %w", actorUID, err) + } + if result.RowsAffected() == 0 { + return store.ErrRuntimeLeaseInvalid + } + return nil +} + +func (p *Persistence) DeleteActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) error { + result, err := p.pool.Exec(ctx, ` + DELETE FROM actor_runtime_leases + WHERE actor_uid = $1 AND token = $2 AND generation = $3`, actorUID, token, generation) + if err != nil { + return fmt.Errorf("deleting runtime lease for actor %q: %w", actorUID, err) + } + if result.RowsAffected() == 0 { + return store.ErrRuntimeLeaseInvalid + } + return nil +} + +func (p *Persistence) ListExpiredActorRuntimeLeases(ctx context.Context, limit int) ([]store.ActorRuntimeLease, error) { + if limit <= 0 { + limit = defaultRuntimeLeaseListLimit + } + rows, err := p.pool.Query(ctx, ` + SELECT actor_uid, actor_atespace, actor_name, token, generation, expires_at + FROM actor_runtime_leases + WHERE expires_at <= clock_timestamp() + AND reclaiming_until <= clock_timestamp() + ORDER BY expires_at, actor_uid + LIMIT $1`, limit) + if err != nil { + return nil, fmt.Errorf("listing expired runtime leases: %w", err) + } + defer rows.Close() + + var leases []store.ActorRuntimeLease + for rows.Next() { + var lease store.ActorRuntimeLease + if err := rows.Scan(&lease.ActorUID, &lease.ActorRef.Atespace, &lease.ActorRef.Name, &lease.Token, &lease.Generation, &lease.ExpiresAt); err != nil { + return nil, fmt.Errorf("scanning expired runtime lease: %w", err) + } + leases = append(leases, lease) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("listing expired runtime leases: %w", err) + } + return leases, nil +} diff --git a/cmd/ateapi/internal/store/atepg/atepg_test.go b/cmd/ateapi/internal/store/atepg/atepg_test.go index a783be83bb..d6282c8bf7 100644 --- a/cmd/ateapi/internal/store/atepg/atepg_test.go +++ b/cmd/ateapi/internal/store/atepg/atepg_test.go @@ -712,6 +712,71 @@ func TestAcquireLease_CleansExpiredLeases(t *testing.T) { } } +func TestActorRuntimeLease_CASAndGeneration(t *testing.T) { + s := setupPostgresPersistence(t) + ctx := context.Background() + ref := resources.ActorRef{Atespace: "team-a", Name: "actor-a"} + first, err := s.IssueActorRuntimeLease(ctx, "actor-uid-1", ref) + if err != nil { + t.Fatalf("IssueActorRuntimeLease: %v", err) + } + if _, err := s.IssueActorRuntimeLease(ctx, "actor-uid-1", ref); !errors.Is(err, store.ErrLeaseConflict) { + t.Fatalf("second IssueActorRuntimeLease = %v, want ErrLeaseConflict", err) + } + if _, err := s.RenewActorRuntimeLease(ctx, first.ActorUID, first.Token, first.Generation+1); !errors.Is(err, store.ErrRuntimeLeaseInvalid) { + t.Fatalf("stale renewal = %v, want ErrRuntimeLeaseInvalid", err) + } + renewed, err := s.RenewActorRuntimeLease(ctx, first.ActorUID, first.Token, first.Generation) + if err != nil { + t.Fatalf("RenewActorRuntimeLease: %v", err) + } + if renewed.Generation != first.Generation || !renewed.ExpiresAt.After(first.ExpiresAt) { + t.Fatalf("renewed lease = %+v, want same generation and later expiry than %+v", renewed, first) + } + if _, err := s.pool.Exec(ctx, `UPDATE actor_runtime_leases SET expires_at = clock_timestamp() - interval '1 second' WHERE actor_uid = $1`, first.ActorUID); err != nil { + t.Fatalf("expire runtime lease: %v", err) + } + if err := s.ClaimExpiredActorRuntimeLease(ctx, first.ActorUID, first.Token, first.Generation); err != nil { + t.Fatalf("ClaimExpiredActorRuntimeLease: %v", err) + } + if _, err := s.RenewActorRuntimeLease(ctx, first.ActorUID, first.Token, first.Generation); !errors.Is(err, store.ErrRuntimeLeaseInvalid) { + t.Fatalf("renewal after reclaim claim = %v, want ErrRuntimeLeaseInvalid", err) + } + if err := s.DeleteActorRuntimeLease(ctx, first.ActorUID, first.Token, first.Generation+1); !errors.Is(err, store.ErrRuntimeLeaseInvalid) { + t.Fatalf("stale delete = %v, want ErrRuntimeLeaseInvalid", err) + } + if err := s.DeleteActorRuntimeLease(ctx, first.ActorUID, first.Token, first.Generation); err != nil { + t.Fatalf("DeleteActorRuntimeLease: %v", err) + } + second, err := s.IssueActorRuntimeLease(ctx, "actor-uid-1", ref) + if err != nil { + t.Fatalf("IssueActorRuntimeLease after delete: %v", err) + } + if second.Generation <= first.Generation { + t.Fatalf("second generation = %d, want greater than %d", second.Generation, first.Generation) + } +} + +func TestAcquireLease_DoesNotCleanActorRuntimeLeases(t *testing.T) { + s := setupPostgresPersistence(t) + ctx := context.Background() + if _, err := s.pool.Exec(ctx, ` + INSERT INTO actor_runtime_leases + (actor_uid, actor_atespace, actor_name, token, generation, expires_at) + VALUES ('runtime-uid', 'team-a', 'actor-a', 'runtime-token', 1, + clock_timestamp() - interval '1 minute')`); err != nil { + t.Fatalf("seeding runtime lease: %v", err) + } + lease, err := s.AcquireLease(ctx, "workflow") + if err != nil { + t.Fatalf("AcquireLease: %v", err) + } + defer lease.Close() + if _, err := s.GetActorRuntimeLease(ctx, "runtime-uid"); err != nil { + t.Fatalf("GetActorRuntimeLease after workflow cleanup: %v", err) + } +} + // TestCreateActor_MissingAtespace_FailedPrecondition exercises the // foreign-key race the doc calls out: CreateActor rejects an actor whose // atespace doesn't exist (including a concurrently-deleted one), with the diff --git a/cmd/ateapi/internal/store/atepg/migrations/000002_actor_runtime_leases.sql b/cmd/ateapi/internal/store/atepg/migrations/000002_actor_runtime_leases.sql new file mode 100644 index 0000000000..b26160e541 --- /dev/null +++ b/cmd/ateapi/internal/store/atepg/migrations/000002_actor_runtime_leases.sql @@ -0,0 +1,32 @@ +-- Copyright 2026 Google LLC +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +-- +goose Up + +-- Runtime ownership is separate from workflow leases: generic lease cleanup +-- must never be able to erase a lease that protects a live actor workload. +CREATE SEQUENCE actor_runtime_lease_generation_seq AS bigint; + +CREATE TABLE actor_runtime_leases ( + actor_uid text PRIMARY KEY, + actor_atespace text NOT NULL, + actor_name text NOT NULL, + token text NOT NULL, + generation bigint NOT NULL, + expires_at timestamptz NOT NULL, + reclaiming_until timestamptz NOT NULL DEFAULT 'epoch' +); + +CREATE INDEX actor_runtime_leases_expires_at_idx + ON actor_runtime_leases (expires_at); diff --git a/cmd/ateapi/internal/store/store.go b/cmd/ateapi/internal/store/store.go index 2d4b0522bd..9a298e670b 100644 --- a/cmd/ateapi/internal/store/store.go +++ b/cmd/ateapi/internal/store/store.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "sync" + "time" "github.com/agent-substrate/substrate/internal/resources" "github.com/agent-substrate/substrate/pkg/proto/ateapipb" @@ -43,6 +44,10 @@ var ( // ErrLeaseConflict indicates that a distributed lease is already held by another client. ErrLeaseConflict = errors.New("persistence: lease conflict") + // ErrRuntimeLeaseInvalid indicates that an actor runtime lease is missing, + // stale, or expired. + ErrRuntimeLeaseInvalid = errors.New("persistence: runtime lease invalid") + // ErrInvalidPageToken indicates that a list page token is malformed or was // issued for a different list operation or scope. ErrInvalidPageToken = errors.New("persistence: invalid page token") @@ -266,6 +271,28 @@ type Interface interface { // held and renewed automatically until the returned Lease is closed. // Returns ErrLeaseConflict if the lease is already held by another client. AcquireLease(ctx context.Context, key string) (*Lease, error) + + // IssueActorRuntimeLease creates the runtime lease for an actor UID. An + // existing row, including an expired row awaiting reclaim, conflicts. + IssueActorRuntimeLease(ctx context.Context, actorUID string, actorRef resources.ActorRef) (*ActorRuntimeLease, error) + + // GetActorRuntimeLease fetches the runtime lease for an actor UID. + GetActorRuntimeLease(ctx context.Context, actorUID string) (*ActorRuntimeLease, error) + + // RenewActorRuntimeLease compares the complete caller tuple and extends the + // lease only while it is still current, unexpired, and not being reclaimed. + RenewActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) (*ActorRuntimeLease, error) + + // ClaimExpiredActorRuntimeLease fences renewal before a reaper terminates + // the workload. A claim expires on its own if the reaper process dies. + ClaimExpiredActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) error + + // DeleteActorRuntimeLease removes a lease only when its token and generation + // still match. It is used after a successful suspend or reclaim. + DeleteActorRuntimeLease(ctx context.Context, actorUID, token string, generation int64) error + + // ListExpiredActorRuntimeLeases returns runtime leases ready for reclaim. + ListExpiredActorRuntimeLeases(ctx context.Context, limit int) ([]ActorRuntimeLease, error) } // Precondition guards an update with the uid and version the caller observed: @@ -391,6 +418,15 @@ type Lease struct { once sync.Once } +// ActorRuntimeLease is durable ownership of a running actor workload. +type ActorRuntimeLease struct { + ActorUID string + ActorRef resources.ActorRef + Token string + Generation int64 + ExpiresAt time.Time +} + // NewLease builds a Lease from its lease context (cancelled on loss or Close) // and the func that stops lease renewal and releases it. func NewLease(ctx context.Context, closeFn func()) *Lease { diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index e35af167cf..cf74e02729 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -199,6 +199,7 @@ func main() { volPlugins := make(map[string]volume.VolumePluginControlPlane) ateletDialer := controlapi.NewAteletDialer(workerPodInformer.GetIndexer(), ateletPodInformer.GetIndexer(), *ateletClientCredBundle, *podIdentityCACerts) controlSrv := controlapi.NewRPCService(persistence, workerCache, sandboxConfigLister, csiDriverConfigLister, storageClassLister, ateletDialer, instruments, *egressGatewayAddress, volPlugins, objectStore) + controlSrv.StartRuntimeLeaseReconciler(shutdownCtx) // Drive stored ActorTemplates through the golden actor flow. templateReconciler := controlapi.NewActorTemplateReconciler(persistence, controlSrv) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index ff7835cc27..1160f779af 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -45,6 +45,7 @@ import ( "github.com/agent-substrate/substrate/internal/childreap" "github.com/agent-substrate/substrate/internal/contextlogging" "github.com/agent-substrate/substrate/internal/imagecache" + "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/otlprelay" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/readyz" @@ -92,9 +93,11 @@ var ( // ingress to. const actorHTTPUpstream = "http://" + ateomnet.ActorVethIP + ":80" -// Workers get a conservative shutdown period. This needs to be significantly less than the K8s -// termination grace period for the ateom. -const workloadGracePeriod = 1 * time.Minute +// workloadGracePeriod is the whole budget for draining the worker on shutdown. +// It needs to stay significantly less than the K8s termination grace period +// for the ateom, so the escalation to SIGKILL happens here rather than as a +// kubelet SIGKILL of ateom itself. +const workloadGracePeriod = 30 * time.Minute // resumeTimeout is the conservative ceiling for unpausing a paused sandbox. const resumeTimeout = 30 * time.Second @@ -491,15 +494,18 @@ func (s *AteomService) gracefulShutdown(ctx context.Context) { // a SIGTERM. s.cancelActiveRestoreOrRunRPC() + // One deadline covers the whole drain. Waiting for the lock and waiting out + // SIGTERM below both run against it, so the two phases split a single grace + // period rather than each getting one: an RPC that burns most of the budget + // leaves the containers only the remainder, and the total stays bounded by + // workloadGracePeriod however the time falls between them. + deadline := time.Now().Add(workloadGracePeriod) + // Attempt to acquire the lock used to serialize ateom RPCs. This will wait for any // pending RPCs to finish (suspend, resume, etc...). After the RPCs finish there // should be no active session. The run / resume was cancelled and the // checkpoint / restore will stop the workload and clear the active session. - // - // In the worst case, these RPCs take almost the entire grace period and then - // fail. We will then proceed to send SIGTERM to the containers and wait for - // them to exit, potentially waiting for 2x the total grace period. - lockCtx, lockCancel := context.WithTimeout(ctx, workloadGracePeriod) + lockCtx, lockCancel := context.WithDeadline(ctx, deadline) defer lockCancel() if !s.lock.LockContext(lockCtx) { @@ -520,7 +526,7 @@ func (s *AteomService) gracefulShutdown(ctx context.Context) { wg.Add(1) go func(containerName string) { defer wg.Done() - if err := s.killContainer(ctx, session, containerName); err != nil { + if err := killContainer(ctx, session.rcmd, containerName, deadline); err != nil { slog.WarnContext(ctx, "Failed to kill container during shutdown", slog.String("container", containerName), slog.Any("err", err)) } }(name) @@ -530,23 +536,39 @@ func (s *AteomService) gracefulShutdown(ctx context.Context) { slog.InfoContext(ctx, "Shutting down") } -// killContainer stops a container by sending SIGTERM, waiting for the grace period, -// and escalating to SIGKILL if necessary. -func (s *AteomService) killContainer(ctx context.Context, session *workloadSession, name string) error { +// containerKillTimeout bounds the post-SIGKILL wait, so a completely broken +// gVisor cannot hold shutdown open indefinitely. It is deliberately not drawn +// from the grace period: by this point the container has already had its +// allowance and the deadline has passed. A var so tests can shorten it. +var containerKillTimeout = 5 * time.Second + +// containerRuntime is the slice of *runsc that graceful shutdown needs. Narrowed +// to an interface so killContainer's SIGTERM-then-SIGKILL escalation can be +// exercised without executing runsc. +type containerRuntime interface { + cmdKill(ctx context.Context, containerName, signal string) error + cmdWait(ctx context.Context, containerName string) error +} + +// killContainer stops a container by sending SIGTERM, waiting until deadline, and +// escalating to SIGKILL if necessary. deadline is the shared drain deadline, so a +// caller that has already spent most of the grace period elsewhere leaves the +// container only what is left of it. +func killContainer(ctx context.Context, rcmd containerRuntime, name string, deadline time.Time) error { // Propagate SIGTERM to the application container so it can save state and close connections. // If the actor installed no SIGTERM handler it terminates immediately. slog.InfoContext(ctx, "Sending SIGTERM to container", slog.String("container", name)) - if err := session.rcmd.cmdKill(ctx, name, "SIGTERM"); err != nil { + if err := rcmd.cmdKill(ctx, name, "SIGTERM"); err != nil { slog.ErrorContext(ctx, "Failed to propagate SIGTERM to container", slog.String("container", name), slog.Any("err", err)) return fmt.Errorf("failed to propagate SIGTERM to container %q: %w", name, err) } done := make(chan error, 1) go func() { - done <- session.rcmd.cmdWait(ctx, name) + done <- rcmd.cmdWait(ctx, name) }() - sigTermCtx, sigTermCtxCancel := context.WithTimeout(ctx, workloadGracePeriod) + sigTermCtx, sigTermCtxCancel := context.WithDeadline(ctx, deadline) defer sigTermCtxCancel() err := waitContainerStop(sigTermCtx, done) @@ -567,15 +589,13 @@ func (s *AteomService) killContainer(ctx context.Context, session *workloadSessi return ctx.Err() } - // sigTermCtx timed out. Send SIGKILL. + // sigTermCtx hit the drain deadline. Send SIGKILL. slog.WarnContext(ctx, "Grace period expired; killing container", slog.String("container", name)) - if err := session.rcmd.cmdKill(ctx, name, "SIGKILL"); err != nil { + if err := rcmd.cmdKill(ctx, name, "SIGKILL"); err != nil { slog.WarnContext(ctx, "Failed to send SIGKILL to container (it might have already exited)", slog.String("container", name), slog.Any("err", err)) } - // Block until the killed container actually exits, but set a short timeout (e.g. 5 seconds) - // to avoid blocking indefinitely if gVisor is completely broken. - killCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + killCtx, cancel := context.WithTimeout(ctx, containerKillTimeout) defer cancel() err = waitContainerStop(killCtx, done) @@ -637,7 +657,7 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // Contract with atelet: // // * Correct runsc version is downloaded and placed on disk. - // * All OCI bundles are set up, including for "pause" container. + // * All OCI bundles are set up, including for the pause container. egress, err := s.prepareActorEgress(ctx, req.GetActorUid(), req.GetEgressGateway()) if err != nil { @@ -687,14 +707,14 @@ func (s *AteomService) RunWorkload(ctx context.Context, req *ateompb.RunWorkload // upper — because mounting is ateom's job (atelet runs with no // capabilities); runsc's gofer resolves the mount in this pod's mount // namespace. - if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), "pause")); err != nil { + if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), ocispec.PauseContainer)); err != nil { return nil, fmt.Errorf("while composing pause rootfs: %w", err) } - containersToDelete = append(containersToDelete, "pause") - if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", nil); err != nil { + containersToDelete = append(containersToDelete, ocispec.PauseContainer) + if err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil); err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) } - if err := rcmd.cmdStart(ctx, os.Stdout, "pause"); err != nil { + if err := rcmd.cmdStart(ctx, os.Stdout, ocispec.PauseContainer); err != nil { return nil, fmt.Errorf("while starting pause container: %w", err) } @@ -773,7 +793,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec if !hasDurableVolumes(req.GetSpec().GetContainers()) { return nil, fmt.Errorf("no durable-dir volumes found for DATA snapshot") } - if err := rcmd.cmdPause(ctx, "pause"); err != nil { + if err := rcmd.cmdPause(ctx, ocispec.PauseContainer); err != nil { return nil, fmt.Errorf("while pausing pause container: %w", err) } tarErr := tarDurableVolumes(ctx, ateompath.DurableDirVolumeMountsDir(req.GetActorUid()), checkpointPath) @@ -782,7 +802,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // fail the resume instantly and leave the sandbox paused forever. resumeCtx, cancelResume := context.WithTimeout(context.WithoutCancel(ctx), resumeTimeout) defer cancelResume() - if err := rcmd.cmdResume(resumeCtx, "pause"); err != nil { + if err := rcmd.cmdResume(resumeCtx, ocispec.PauseContainer); err != nil { return nil, fmt.Errorf("while resuming pause container: %w", err) } if tarErr != nil { @@ -791,7 +811,7 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: // Checkpoint pause container (root of the sandbox) // TODO: Consider pause -> tar -> resume -> checkpoint order for better failure handling. - if err := rcmd.cmdCheckpoint(ctx, "pause", checkpointPath); err != nil { + if err := rcmd.cmdCheckpoint(ctx, ocispec.PauseContainer, checkpointPath); err != nil { return nil, fmt.Errorf("while checkpointing pause: %w", err) } if hasDurableVolumes(req.GetSpec().GetContainers()) { @@ -860,15 +880,15 @@ func (r *runsc) stopContainers(ctx context.Context, containers []*ateompb.Contai _ = r.cmdKill(ctx, ctr.GetName(), "SIGKILL") _ = r.cmdWait(ctx, ctr.GetName()) } - _ = r.cmdKill(ctx, "pause", "SIGKILL") - _ = r.cmdWait(ctx, "pause") + _ = r.cmdKill(ctx, ocispec.PauseContainer, "SIGKILL") + _ = r.cmdWait(ctx, ocispec.PauseContainer) } func (r *runsc) cleanupContainers(ctx context.Context, containers []*ateompb.Container) error { // Check state of all containers to mimic containerd. // // Without this, `runsc delete` occasionally throws an error. - if err := r.cmdState(ctx, "pause"); err != nil { + if err := r.cmdState(ctx, ocispec.PauseContainer); err != nil { return fmt.Errorf("while checking state of pause container: %w", err) } for _, ctr := range containers { @@ -883,7 +903,7 @@ func (r *runsc) cleanupContainers(ctx context.Context, containers []*ateompb.Con } } - if err := r.cmdDelete(ctx, "pause"); err != nil { + if err := r.cmdDelete(ctx, ocispec.PauseContainer); err != nil { return fmt.Errorf("while deleting pause container: %w", err) } @@ -915,7 +935,7 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // Contract with atelet: // // * Correct runsc version is downloaded and placed on disk. - // * All OCI bundles are set up, including for "pause" container. + // * All OCI bundles are set up, including for the pause container. // * Checkpoint downloaded and placed on disk egress, err := s.prepareActorEgress(ctx, req.GetActorUid(), req.GetEgressGateway()) @@ -967,27 +987,27 @@ func (s *AteomService) RestoreWorkload(ctx context.Context, req *ateompb.Restore // Compose the pause rootfs before create (see RunWorkload). runsc restore // only needs the rootfs to hold the correct content; whether it came from // an untar or an overlay of cached layers is transparent to it. - if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), "pause")); err != nil { + if err := imagecache.SetupBundleRootfs(ateompath.OCIBundlePath(req.GetActorUid(), ocispec.PauseContainer)); err != nil { return nil, fmt.Errorf("while composing pause rootfs: %w", err) } switch req.GetScope() { case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: // Create and start pause container (cold boot with durable-dir volumes restored) - containersToDelete = append(containersToDelete, "pause") - if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", nil); err != nil { + containersToDelete = append(containersToDelete, ocispec.PauseContainer) + if err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil); err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) } - if err := rcmd.cmdStart(ctx, os.Stdout, "pause"); err != nil { + if err := rcmd.cmdStart(ctx, os.Stdout, ocispec.PauseContainer); err != nil { return nil, fmt.Errorf("while starting pause container: %w", err) } case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN: // Create and restore pause container - containersToDelete = append(containersToDelete, "pause") - if err := rcmd.cmdCreate(ctx, os.Stdout, "pause", nil); err != nil { + containersToDelete = append(containersToDelete, ocispec.PauseContainer) + if err := rcmd.cmdCreate(ctx, os.Stdout, ocispec.PauseContainer, nil); err != nil { return nil, fmt.Errorf("while creating pause container: %w", err) } - if err := rcmd.cmdRestore(ctx, os.Stdout, "pause", checkpointDir); err != nil { + if err := rcmd.cmdRestore(ctx, os.Stdout, ocispec.PauseContainer, checkpointDir); err != nil { return nil, fmt.Errorf("while restoring pause container: %w", err) } default: diff --git a/cmd/ateom-gvisor/runsc_test.go b/cmd/ateom-gvisor/runsc_test.go index c6e7d77aef..c46373cd70 100644 --- a/cmd/ateom-gvisor/runsc_test.go +++ b/cmd/ateom-gvisor/runsc_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/ocispec" ) func TestKillArgs(t *testing.T) { @@ -70,13 +71,13 @@ func TestPauseArgs(t *testing.T) { actorUID: "test-actor-123", } - got := r.pauseArgs("pause") + got := r.pauseArgs(ocispec.PauseContainer) want := []string{ "-log-format", "json", "--alsologtostderr", "-root", ateompath.RunSCStateDir("test-actor-123"), "pause", - "pause", + ocispec.PauseContainer, } if !reflect.DeepEqual(got, want) { @@ -90,13 +91,13 @@ func TestResumeArgs(t *testing.T) { actorUID: "test-actor-123", } - got := r.resumeArgs("pause") + got := r.resumeArgs(ocispec.PauseContainer) want := []string{ "-log-format", "json", "--alsologtostderr", "-root", ateompath.RunSCStateDir("test-actor-123"), "resume", - "pause", + ocispec.PauseContainer, } if !reflect.DeepEqual(got, want) { diff --git a/cmd/ateom-gvisor/shutdown_test.go b/cmd/ateom-gvisor/shutdown_test.go new file mode 100644 index 0000000000..b23b3c5672 --- /dev/null +++ b/cmd/ateom-gvisor/shutdown_test.go @@ -0,0 +1,195 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "reflect" + "strings" + "sync" + "testing" + "time" +) + +// fakeRuntime stands in for *runsc. It records the signals killContainer +// delivers and unblocks cmdWait when the container is configured to die on one +// of them. +type fakeRuntime struct { + mu sync.Mutex + signals []string + + // exitOn is the signal that makes the container exit. Empty means it never + // does, which is how the wedged-sandbox case is set up. + exitOn string + // waitErr is what cmdWait reports once it unblocks, standing in for a + // `runsc wait` that failed rather than a container that stopped. + waitErr error + // killErr maps a signal to the error cmdKill returns for it. + killErr map[string]error + + exited chan struct{} + exitedOnce sync.Once +} + +func newFakeRuntime(exitOn string, waitErr error, killErr map[string]error) *fakeRuntime { + return &fakeRuntime{exitOn: exitOn, waitErr: waitErr, killErr: killErr, exited: make(chan struct{})} +} + +func (f *fakeRuntime) cmdKill(_ context.Context, _, signal string) error { + f.mu.Lock() + f.signals = append(f.signals, signal) + f.mu.Unlock() + + if f.exitOn == signal { + f.exitedOnce.Do(func() { close(f.exited) }) + } + return f.killErr[signal] +} + +func (f *fakeRuntime) cmdWait(ctx context.Context, _ string) error { + select { + case <-f.exited: + return f.waitErr + case <-ctx.Done(): + return ctx.Err() + } +} + +func (f *fakeRuntime) sentSignals() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.signals...) +} + +// TestKillContainer covers the SIGTERM-then-SIGKILL escalation against the +// shared drain deadline. The deadlines here are milliseconds rather than the +// production grace period, which is what the package-level vars are for. +func TestKillContainer(t *testing.T) { + tests := []struct { + name string + // exitOn, waitErr and killErr configure the fake runtime. + exitOn string + waitErr error + killErr map[string]error + // deadlineIn is the drain deadline relative to the start of the call. A + // negative value is a deadline the caller has already spent elsewhere. + deadlineIn time.Duration + wantSignals []string + wantErr string + }{ + { + name: "container exits on SIGTERM inside the deadline", + exitOn: "SIGTERM", + deadlineIn: time.Minute, + wantSignals: []string{"SIGTERM"}, + }, + { + name: "container ignoring SIGTERM is killed at the deadline", + exitOn: "SIGKILL", + deadlineIn: 20 * time.Millisecond, + wantSignals: []string{"SIGTERM", "SIGKILL"}, + }, + { + // The lock wait ate the whole grace period, so the container gets + // SIGTERM and no time at all before the escalation. + name: "deadline already spent leaves no grace", + exitOn: "SIGKILL", + deadlineIn: -time.Second, + wantSignals: []string{"SIGTERM", "SIGKILL"}, + }, + { + name: "container surviving SIGKILL is reported", + deadlineIn: 20 * time.Millisecond, + wantSignals: []string{"SIGTERM", "SIGKILL"}, + wantErr: "failed to exit even after SIGKILL", + }, + { + name: "undeliverable SIGTERM does not escalate", + killErr: map[string]error{"SIGTERM": errors.New("sandbox gone")}, + deadlineIn: time.Minute, + wantSignals: []string{"SIGTERM"}, + wantErr: "failed to propagate SIGTERM", + }, + { + // A failing `runsc wait` says nothing about the container, so the + // caller is told rather than the container killed. + name: "failed wait does not escalate", + exitOn: "SIGTERM", + waitErr: errors.New("runsc wait exploded"), + deadlineIn: time.Minute, + wantSignals: []string{"SIGTERM"}, + wantErr: "wait failed", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + origKillTimeout := containerKillTimeout + containerKillTimeout = 20 * time.Millisecond + t.Cleanup(func() { containerKillTimeout = origKillTimeout }) + + // A cancelled context is what releases a cmdWait the fake never + // unblocks, so the wait goroutine does not outlive the test. + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + f := newFakeRuntime(tc.exitOn, tc.waitErr, tc.killErr) + err := killContainer(ctx, f, "counter", time.Now().Add(tc.deadlineIn)) + + switch { + case tc.wantErr == "" && err != nil: + t.Errorf("killContainer() = %v, want nil", err) + case tc.wantErr != "" && err == nil: + t.Errorf("killContainer() = nil, want error containing %q", tc.wantErr) + case tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr): + t.Errorf("killContainer() = %v, want error containing %q", err, tc.wantErr) + } + + if got := f.sentSignals(); !reflect.DeepEqual(got, tc.wantSignals) { + t.Errorf("signals delivered = %v, want %v", got, tc.wantSignals) + } + }) + } +} + +// TestKillContainerHonorsParentCancellation asserts that a cancelled shutdown +// context stops the drain instead of escalating: ateom is going away anyway, and +// the containers go down with the pod. +func TestKillContainerHonorsParentCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + f := newFakeRuntime("", nil, nil) + + // Cancel once SIGTERM has been delivered and the wait is under way. + go func() { + for { + if len(f.sentSignals()) > 0 { + cancel() + return + } + time.Sleep(time.Millisecond) + } + }() + + err := killContainer(ctx, f, "counter", time.Now().Add(time.Minute)) + if !errors.Is(err, context.Canceled) { + t.Errorf("killContainer() = %v, want context.Canceled", err) + } + if got := f.sentSignals(); !reflect.DeepEqual(got, []string{"SIGTERM"}) { + t.Errorf("signals delivered = %v, want [SIGTERM]", got) + } +} diff --git a/cmd/ateom-gvisor/stats.go b/cmd/ateom-gvisor/stats.go index dd71895177..d17b13f24e 100644 --- a/cmd/ateom-gvisor/stats.go +++ b/cmd/ateom-gvisor/stats.go @@ -27,6 +27,7 @@ import ( "google.golang.org/grpc/status" "github.com/agent-substrate/substrate/cmd/ateom-gvisor/internal/cgroupstats" + "github.com/agent-substrate/substrate/internal/ocispec" "github.com/agent-substrate/substrate/internal/proto/ateompb" "github.com/agent-substrate/substrate/internal/resources" ) @@ -44,7 +45,7 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // the sentry. runsc starts that process from the root container's create and // from inside that container's cgroup — container.createRoot wraps the sandbox // and gofer spawn in cgroup.RunInCgroup — so the sentry lands in the leaf of -// "pause", the first container RunWorkload and RestoreWorkload create. +// the pause container, the first one RunWorkload and RestoreWorkload create. // // The leaf is a direct child of the delegated scope rather than of ateom's own // cgroup, because runsc resolves cgroupsPath against the parent of the cgroup @@ -70,7 +71,7 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // What the leaf holds besides the actor's own work: the sentry's own overhead // (its Go heap, page tables, netstack) and the gofers. Process listings taken // on a live node in #161 put runsc-sandbox and both gofers — the pause -// container's and the actor container's — in the "pause" cgroup. Those runs +// container's and the actor container's — in the pause cgroup. Those runs // predate #496, so they establish the leaf name and the fact that everything // lands in one leaf, not the absolute path, which #496's delegation moved under // the pod scope. @@ -86,7 +87,7 @@ const defaultCgroupRoot = "/sys/fs/cgroup" // The name has to agree with the cgroupsPath convention in // ocispec.ShapeGVisor, which is "/" + containerName relative to the same // scope. -const sandboxCgroupContainer = "pause" +const sandboxCgroupContainer = ocispec.PauseContainer // GetWorkloadStats implements ateompb.Ateom/GetWorkloadStats. // diff --git a/cmd/ateom-microvm/shutdown.go b/cmd/ateom-microvm/shutdown.go index 8c40c5b0f5..c784754c29 100644 --- a/cmd/ateom-microvm/shutdown.go +++ b/cmd/ateom-microvm/shutdown.go @@ -34,18 +34,22 @@ import ( "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" ) -const ( - // workloadGracePeriod is how long a guest workload gets to handle SIGTERM and - // exit on its own before ateom escalates to SIGKILL. Matches ateom-gvisor, and - // is deliberately shorter than the pod's own termination grace period so the - // escalation happens here rather than as a kubelet SIGKILL of ateom itself. - workloadGracePeriod = 1 * time.Minute +// workloadGracePeriod is the whole budget for draining the worker on shutdown: +// waiting for an in-flight RPC to release the lock and letting the guest +// workloads handle SIGTERM both draw on it, and ateom escalates to SIGKILL once +// it is gone. Matches ateom-gvisor, and is deliberately shorter than the pod's +// own termination grace period — 3600s, set by +// workerTerminationGracePeriodSeconds in cmd/atecontroller — so the escalation +// happens here rather than as a kubelet SIGKILL of ateom itself. +const workloadGracePeriod = 30 * time.Minute - // workloadKillTimeout bounds the post-SIGKILL wait. The VM teardown that - // follows is what ultimately guarantees the workload is gone, so a wedged - // kata-agent must not hold shutdown open past this. - workloadKillTimeout = 5 * time.Second +// workloadKillTimeout bounds the post-SIGKILL wait. The VM teardown that +// follows is what ultimately guarantees the workload is gone, so a wedged +// kata-agent must not hold shutdown open past this. +// A var so tests can shorten it. +var workloadKillTimeout = 5 * time.Second +const ( // signalDeliveryTimeout bounds one SignalProcess round-trip. Delivering a signal // is a local ttrpc call that returns in microseconds; if it has not come back by // now the agent is not answering, and waiting longer will not change that. @@ -72,12 +76,17 @@ func (s *AteomService) gracefulShutdown(ctx context.Context) { // SIGTERM the guest it just produced is strictly worse than aborting it. s.cancelActiveRestoreOrRunRPC() + // One deadline covers the whole drain. Waiting for the lock and waiting out + // SIGTERM below both run against it, so the two phases split a single grace + // period rather than each getting one: an RPC that burns most of the budget + // leaves the workloads only the remainder, and the total stays bounded by + // workloadGracePeriod however the time falls between them. + deadline := time.Now().Add(workloadGracePeriod) + // Wait for whatever still holds lock — a suspend, a resume — to finish, but - // only for the grace period. In the worst case that RPC burns nearly all of it - // and then fails, and the stop below spends another grace period on top; that - // is bounded well inside the pod's own termination grace period, and is the - // price of not truncating an RPC that may be saving the actor's state. - lockCtx, lockCancel := context.WithTimeout(ctx, workloadGracePeriod) + // not past the deadline. Letting it run that long is the price of not + // truncating an RPC that may be saving the actor's state. + lockCtx, lockCancel := context.WithDeadline(ctx, deadline) defer lockCancel() if !s.lock.LockContext(lockCtx) { slog.ErrorContext(ctx, "Failed to acquire lock during graceful shutdown; another RPC is still running") @@ -105,9 +114,20 @@ func (s *AteomService) gracefulShutdown(ctx context.Context) { return } + // Drain the actors concurrently, for the same reason their workloads are + // drained concurrently below: they share one deadline, so in series the first + // actor's wait would come out of every later one's allowance and the last + // would be SIGKILLed with no grace at all. + var wg sync.WaitGroup for _, t := range targets { - gracefullyStopActor(ctx, t) + wg.Add(1) + go func(t drainTarget) { + defer wg.Done() + gracefullyStopActor(ctx, t, deadline) + }(t) } + wg.Wait() + slog.InfoContext(ctx, "Shutting down") } @@ -127,9 +147,18 @@ type drainTarget struct { workloadIDs []string } +// guestAgent is the slice of *kata.AgentClient the drain needs: signal a guest +// process and wait for it to exit. Narrowed to an interface so +// stopGuestWorkload's SIGTERM-then-SIGKILL escalation can be exercised without a +// running VM. +type guestAgent interface { + SignalProcess(ctx context.Context, containerID, execID string, signal uint32) error + WaitProcess(ctx context.Context, containerID, execID string) (int32, error) +} + // gracefullyStopActor signals the actor's guest workloads with SIGTERM and waits -// out the grace period, escalating to SIGKILL. -func gracefullyStopActor(ctx context.Context, t drainTarget) { +// until deadline, escalating to SIGKILL. +func gracefullyStopActor(ctx context.Context, t drainTarget, deadline time.Time) { id := t.id // Obtain a kata-agent client to signal the guest: reuse the log-forwarding @@ -148,15 +177,15 @@ func gracefullyStopActor(ctx context.Context, t drainTarget) { agent, dialed = a, a } - // Stop the workloads concurrently. Each one is entitled to the full grace - // period, so stopping them in series would multiply it by the container - // count and overrun the pod's own termination grace period. + // Stop the workloads concurrently. They share one deadline, so stopping them + // in series would spend the first workload's wait out of every later one's + // allowance and leave the last with none. var wg sync.WaitGroup for _, wid := range t.workloadIDs { wg.Add(1) go func(wid string) { defer wg.Done() - if err := stopGuestWorkload(ctx, agent, id, wid); err != nil { + if err := stopGuestWorkload(ctx, agent, id, wid, deadline); err != nil { slog.WarnContext(ctx, "Failed to stop guest workload during shutdown", slog.String("id", id), slog.String("workload", wid), slog.Any("err", err)) } }(wid) @@ -167,9 +196,11 @@ func gracefullyStopActor(ctx context.Context, t drainTarget) { } } -// stopGuestWorkload stops one guest workload, wait out workloadGracePeriod, then -// escalate to SIGKILL and wait a bounded time for the kill to land. -func stopGuestWorkload(ctx context.Context, agent *kata.AgentClient, id, wid string) error { +// stopGuestWorkload stops one guest workload, waits until deadline, then +// escalates to SIGKILL and waits a bounded time for the kill to land. deadline is +// the shared drain deadline, so a caller that has already spent most of the grace +// period elsewhere leaves the workload only what is left of it. +func stopGuestWorkload(ctx context.Context, agent guestAgent, id, wid string, deadline time.Time) error { // Propagate SIGTERM so the actor can save state and close connections. // An actor that installed no handler terminates immediately. slog.InfoContext(ctx, "Sending SIGTERM to guest workload", slog.String("id", id), slog.String("workload", wid)) @@ -190,7 +221,7 @@ func stopGuestWorkload(ctx context.Context, agent *kata.AgentClient, id, wid str done <- err }() - termCtx, termCancel := context.WithTimeout(ctx, workloadGracePeriod) + termCtx, termCancel := context.WithDeadline(ctx, deadline) defer termCancel() err := waitWorkloadStop(termCtx, done) if err == nil { @@ -216,7 +247,9 @@ func stopGuestWorkload(ctx context.Context, agent *kata.AgentClient, id, wid str return ctx.Err() } - slog.WarnContext(ctx, "Grace period expired; killing guest workload", slog.String("id", id), slog.String("workload", wid), slog.Duration("grace", workloadGracePeriod)) + // The deadline, not the configured grace period: the lock wait may have eaten + // part of the budget before the workload ever saw SIGTERM. + slog.WarnContext(ctx, "Grace period expired; killing guest workload", slog.String("id", id), slog.String("workload", wid), slog.Time("deadline", deadline)) if err := signalWorkload(ctx, agent, wid, syscall.SIGKILL); err != nil { slog.WarnContext(ctx, "Failed to SIGKILL guest workload (it might have already exited)", slog.String("id", id), slog.String("workload", wid), slog.Any("err", err)) } @@ -244,7 +277,7 @@ func stopGuestWorkload(ctx context.Context, agent *kata.AgentClient, id, wid str // land mid-drain — leaves the unix socket to CH perfectly healthy while the agent // never answers, and an unbounded call there would hang until the kubelet's // SIGKILL at the end of the pod's termination grace period. -func signalWorkload(ctx context.Context, agent *kata.AgentClient, wid string, sig syscall.Signal) error { +func signalWorkload(ctx context.Context, agent guestAgent, wid string, sig syscall.Signal) error { sigCtx, cancel := context.WithTimeout(ctx, signalDeliveryTimeout) defer cancel() return agent.SignalProcess(sigCtx, wid, wid, uint32(sig)) diff --git a/cmd/ateom-microvm/shutdown_test.go b/cmd/ateom-microvm/shutdown_test.go new file mode 100644 index 0000000000..7a0b673ba6 --- /dev/null +++ b/cmd/ateom-microvm/shutdown_test.go @@ -0,0 +1,197 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "reflect" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +// fakeGuestAgent stands in for *kata.AgentClient. It records the signals +// stopGuestWorkload delivers and unblocks WaitProcess when the workload is +// configured to die on one of them. +type fakeGuestAgent struct { + mu sync.Mutex + signals []syscall.Signal + + // exitOn is the signal that makes the workload exit. Zero means it never + // does, which is how the wedged-guest case is set up. + exitOn syscall.Signal + // waitErr is what WaitProcess reports once it unblocks, standing in for a + // dead agent connection rather than a process that stopped. + waitErr error + // signalErr maps a signal to the error SignalProcess returns for it. + signalErr map[syscall.Signal]error + + exited chan struct{} + exitedOnce sync.Once +} + +func newFakeGuestAgent(exitOn syscall.Signal, waitErr error, signalErr map[syscall.Signal]error) *fakeGuestAgent { + return &fakeGuestAgent{exitOn: exitOn, waitErr: waitErr, signalErr: signalErr, exited: make(chan struct{})} +} + +func (f *fakeGuestAgent) SignalProcess(_ context.Context, _, _ string, signal uint32) error { + sig := syscall.Signal(signal) + + f.mu.Lock() + f.signals = append(f.signals, sig) + f.mu.Unlock() + + if f.exitOn != 0 && f.exitOn == sig { + f.exitedOnce.Do(func() { close(f.exited) }) + } + return f.signalErr[sig] +} + +func (f *fakeGuestAgent) WaitProcess(ctx context.Context, _, _ string) (int32, error) { + select { + case <-f.exited: + return 0, f.waitErr + case <-ctx.Done(): + return 0, ctx.Err() + } +} + +func (f *fakeGuestAgent) sentSignals() []syscall.Signal { + f.mu.Lock() + defer f.mu.Unlock() + return append([]syscall.Signal(nil), f.signals...) +} + +// TestStopGuestWorkload covers the SIGTERM-then-SIGKILL escalation against the +// shared drain deadline. The deadlines here are milliseconds rather than the +// production grace period, which is what the package-level vars are for. +func TestStopGuestWorkload(t *testing.T) { + tests := []struct { + name string + // exitOn, waitErr and signalErr configure the fake agent. + exitOn syscall.Signal + waitErr error + signalErr map[syscall.Signal]error + // deadlineIn is the drain deadline relative to the start of the call. A + // negative value is a deadline the caller has already spent elsewhere. + deadlineIn time.Duration + wantSignals []syscall.Signal + wantErr string + }{ + { + name: "workload exits on SIGTERM inside the deadline", + exitOn: syscall.SIGTERM, + deadlineIn: time.Minute, + wantSignals: []syscall.Signal{syscall.SIGTERM}, + }, + { + name: "workload ignoring SIGTERM is killed at the deadline", + exitOn: syscall.SIGKILL, + deadlineIn: 20 * time.Millisecond, + wantSignals: []syscall.Signal{syscall.SIGTERM, syscall.SIGKILL}, + }, + { + // The lock wait ate the whole grace period, so the workload gets + // SIGTERM and no time at all before the escalation. + name: "deadline already spent leaves no grace", + exitOn: syscall.SIGKILL, + deadlineIn: -time.Second, + wantSignals: []syscall.Signal{syscall.SIGTERM, syscall.SIGKILL}, + }, + { + name: "workload surviving SIGKILL is reported", + deadlineIn: 20 * time.Millisecond, + wantSignals: []syscall.Signal{syscall.SIGTERM, syscall.SIGKILL}, + wantErr: "failed to exit even after SIGKILL", + }, + { + name: "undeliverable SIGTERM does not escalate", + signalErr: map[syscall.Signal]error{syscall.SIGTERM: errors.New("ttrpc closed")}, + deadlineIn: time.Minute, + wantSignals: []syscall.Signal{syscall.SIGTERM}, + wantErr: "while propagating SIGTERM to workload", + }, + { + // A WaitProcess that errors is a dead agent connection, not a bad + // exit, so liveness is unknown and the caller is told rather than + // the workload killed. + name: "failed wait does not escalate", + exitOn: syscall.SIGTERM, + waitErr: errors.New("ttrpc: closed"), + deadlineIn: time.Minute, + wantSignals: []syscall.Signal{syscall.SIGTERM}, + wantErr: `while waiting for workload "counter" to exit`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + origKillTimeout := workloadKillTimeout + workloadKillTimeout = 20 * time.Millisecond + t.Cleanup(func() { workloadKillTimeout = origKillTimeout }) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + f := newFakeGuestAgent(tc.exitOn, tc.waitErr, tc.signalErr) + err := stopGuestWorkload(ctx, f, "actor-1", "counter", time.Now().Add(tc.deadlineIn)) + + switch { + case tc.wantErr == "" && err != nil: + t.Errorf("stopGuestWorkload() = %v, want nil", err) + case tc.wantErr != "" && err == nil: + t.Errorf("stopGuestWorkload() = nil, want error containing %q", tc.wantErr) + case tc.wantErr != "" && !strings.Contains(err.Error(), tc.wantErr): + t.Errorf("stopGuestWorkload() = %v, want error containing %q", err, tc.wantErr) + } + + if got := f.sentSignals(); !reflect.DeepEqual(got, tc.wantSignals) { + t.Errorf("signals delivered = %v, want %v", got, tc.wantSignals) + } + }) + } +} + +// TestStopGuestWorkloadHonorsParentCancellation asserts that a cancelled shutdown +// context stops the drain instead of escalating: ateom is going away anyway, and +// the guest goes down with the VM. +func TestStopGuestWorkloadHonorsParentCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + f := newFakeGuestAgent(0, nil, nil) + + // Cancel once SIGTERM has been delivered and the wait is under way. + go func() { + for { + if len(f.sentSignals()) > 0 { + cancel() + return + } + time.Sleep(time.Millisecond) + } + }() + + err := stopGuestWorkload(ctx, f, "actor-1", "counter", time.Now().Add(time.Minute)) + if !errors.Is(err, context.Canceled) { + t.Errorf("stopGuestWorkload() = %v, want context.Canceled", err) + } + if got := f.sentSignals(); !reflect.DeepEqual(got, []syscall.Signal{syscall.SIGTERM}) { + t.Errorf("signals delivered = %v, want [SIGTERM]", got) + } +} diff --git a/docs/upgrade.md b/docs/upgrade.md index 970a60d8c4..80a7aa7255 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -27,6 +27,12 @@ a version suffix, and the installed ate-api-server serves install instead, because its DaemonSet selector cannot be changed in place. +Nothing drains worker nodes on its own: node auto-upgrade is off on +every pool that runs workers and none of them is spot or preemptible, +as the +[Create Cluster warning](../tools/setup-gcp/README.md#2-create-cluster) +requires. + Actor snapshots are readable by both the old and the new build. An actor can therefore suspend on one version and resume on the other in either direction, which is what lets the two versions serve side by @@ -48,7 +54,19 @@ Three things break an upgrade. same node, and old workers end up next to the new atelet: exactly the version skew the roll exists to prevent. 2. **Do not edit a serving worker pool.** The controller would roll - the pool's Deployment straight through live actors. + the pool's Deployment straight through live actors. A deleted + worker pod does go through the eviction path: `SIGTERM` is + forwarded into the actor's containers and the control plane keeps + accepting a suspend for about 60 seconds, so an actor suspended + inside that window saves its state and stays resumable. Handling + `SIGTERM` by exiting cleanly is not enough on its own; the suspend + has to reach the control plane and finish. An actor still awake + when the window closes moves to `ACTOR_STATE_CRASHED`, which is + terminal: `resume` and `suspend` are both refused, there is no + recover verb, and the snapshot the actor still holds cannot be + used to start it. It has to be deleted and recreated, losing its + state. The same applies to scaling a serving pool down, which + removes pods without suspending the actors on them. 3. **(If on GKE) Do not touch the node pool's label until every node is rolled.** A pool label update applies in place to every node in the pool, so the whole fleet flips at once, with no drain and no diff --git a/internal/e2e/suites/demo/termination_test.go b/internal/e2e/suites/demo/termination_test.go index 59b5cf3ba0..4f00f76c72 100644 --- a/internal/e2e/suites/demo/termination_test.go +++ b/internal/e2e/suites/demo/termination_test.go @@ -139,91 +139,6 @@ func waitForWorkerRemoved(ctx context.Context, t *testing.T, clients *e2e.Client } } -// TestGracefulWorkerTerminationTimeout exercises the case where the workload -// container hangs (exceeds the 1-minute workloadGracePeriod) during SIGTERM. -// The ateom is expected to SIGKILL the container, letting the control plane -// mark the worker removed and the actor CRASHED. Runs against both runtimes. -func TestGracefulWorkerTerminationTimeout(t *testing.T) { - nsObj := e2e.CreateNamespace(t) - - ctx := context.Background() - clients := e2e.GetClients() - - at, err := createActorTemplate(ctx, t, clients, nsObj, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.SnapshotContentScope_SNAPSHOT_CONTENT_SCOPE_FULL, ateapipb.ResumeSource_RESUME_SOURCE_COLD_BOOT) - if err != nil { - t.Fatalf("failed to initialize ActorTemplate: %v", err) - } - - actorID := "graceful-term-timeout-" + nsObj.Name - if _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{ - Actor: &ateapipb.Actor{ - Metadata: &ateapipb.ResourceMetadata{Atespace: demoAtespace, Name: actorID}, - ActorTemplate: e2e.TemplateRef(at), - }, - }); err != nil { - t.Fatalf("failed to create Actor: %v", err) - } - defer func() { - _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, - }) - }() - - // Bring the actor up on a worker so it is bound to a pod. - if _, err := e2e.ResumeActorAwaitCapacity(t, ctx, clients, &ateapipb.ResumeActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, - }); err != nil { - t.Fatalf("failed to resume Actor: %v", err) - } - waitForActorState(ctx, t, clients, actorID, ateapipb.ActorState_ACTOR_STATE_RUNNING) - - // Set the sigterm sleep interval to 90 seconds (longer than the 1-minute grace period). - if _, err := callActorPath(t, resources.ActorRef{Atespace: demoAtespace, Name: actorID}, "GET", "/set-sigterm-sleep?duration=90"); err != nil { - t.Fatalf("failed to set sigterm sleep: %v", err) - } - - running, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, - }) - if err != nil { - t.Fatalf("failed to get running Actor: %v", err) - } - podNS := running.GetStatus().GetWorkerAssignment().GetWorkerNamespace() - podName := running.GetStatus().GetWorkerAssignment().GetWorkerPod() - if podNS == "" || podName == "" { - t.Fatalf("running actor has no bound worker pod: ns=%q name=%q", podNS, podName) - } - t.Logf("Actor %q bound to worker pod %s/%s", actorID, podNS, podName) - - // Evict the worker pod. The kubelet sends SIGTERM to ateom, which propagates - // it into the sandbox; the container hangs, triggering the 1-minute timeout, - // followed by SIGKILL by ateom. - if err := clients.K8s.CoreV1().Pods(podNS).Delete(ctx, podName, metav1.DeleteOptions{}); err != nil { - t.Fatalf("failed to delete worker pod %s/%s: %v", podNS, podName, err) - } - - // The worker record must eventually be removed once the pod is gone. - // Since there is a 1-minute timeout + up to 5s SIGKILL wait, we need a - // larger timeout (e.g. 120 seconds). - if err := waitForWorkerRemoved(ctx, t, clients, podName, 120*time.Second); err != nil { - t.Fatalf("worker %s not removed after pod deletion: %v", podName, err) - } - - // Verify the actor lands in ACTOR_STATE_CRASHED. - waitForActorStateWithTimeout(ctx, t, clients, actorID, ateapipb.ActorState_ACTOR_STATE_CRASHED, 120*time.Second) - - // Verify the pod assignment was cleared. - actor, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ - Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorID}, - }) - if err != nil { - t.Fatalf("failed to get actor: %v", err) - } - if pod := actor.GetStatus().GetWorkerAssignment().GetWorkerPod(); pod != "" { - t.Errorf("actor still bound to worker pod %q, expected empty", pod) - } -} - // TestGracefulWorkerTerminationSuspend exercises the case where a worker pod is // deleted (evicted), and while the container is in its SIGTERM shutdown phase, // we initiate a suspend. Suspend should succeed. diff --git a/internal/ocispec/gvisor.go b/internal/ocispec/gvisor.go index 62608874a3..a62cd13f94 100644 --- a/internal/ocispec/gvisor.go +++ b/internal/ocispec/gvisor.go @@ -21,8 +21,10 @@ import ( "github.com/opencontainers/runtime-spec/specs-go" ) -// PauseContainer is the name of the sandbox root container. -const PauseContainer = "pause" +// PauseContainer is the name of the sandbox root container. The underscore +// keeps it outside the k8s-short-name an ActorTemplate container +// name is drawn from, so no actor container can collide with it. +const PauseContainer = "_pause" // resolvConf is the host resolver config bound into the sandbox. const resolvConf = "/etc/resolv.conf" diff --git a/pkg/proto/ateapipb/ateapi.pb.go b/pkg/proto/ateapipb/ateapi.pb.go index 13db70e1fb..217f05d9a9 100644 --- a/pkg/proto/ateapipb/ateapi.pb.go +++ b/pkg/proto/ateapipb/ateapi.pb.go @@ -4416,6 +4416,63 @@ func (x *SuspendActorResponse) GetActor() *Actor { return nil } +type SuspendActorWithLeaseRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // +k8s:required + // +k8s:subfield(atespace)=+k8s:required + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + // The runtime lease returned by ResumeActor. + // + // +k8s:required + Lease *ActorLease `protobuf:"bytes,2,opt,name=lease,proto3" json:"lease,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SuspendActorWithLeaseRequest) Reset() { + *x = SuspendActorWithLeaseRequest{} + mi := &file_ateapi_proto_msgTypes[57] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SuspendActorWithLeaseRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuspendActorWithLeaseRequest) ProtoMessage() {} + +func (x *SuspendActorWithLeaseRequest) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[57] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuspendActorWithLeaseRequest.ProtoReflect.Descriptor instead. +func (*SuspendActorWithLeaseRequest) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{57} +} + +func (x *SuspendActorWithLeaseRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +func (x *SuspendActorWithLeaseRequest) GetLease() *ActorLease { + if x != nil { + return x.Lease + } + return nil +} + type PauseActorRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // +k8s:required @@ -4427,7 +4484,7 @@ type PauseActorRequest struct { func (x *PauseActorRequest) Reset() { *x = PauseActorRequest{} - mi := &file_ateapi_proto_msgTypes[57] + mi := &file_ateapi_proto_msgTypes[58] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4439,7 +4496,7 @@ func (x *PauseActorRequest) String() string { func (*PauseActorRequest) ProtoMessage() {} func (x *PauseActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[57] + mi := &file_ateapi_proto_msgTypes[58] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4452,7 +4509,7 @@ func (x *PauseActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use PauseActorRequest.ProtoReflect.Descriptor instead. func (*PauseActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{57} + return file_ateapi_proto_rawDescGZIP(), []int{58} } func (x *PauseActorRequest) GetActor() *ObjectRef { @@ -4471,7 +4528,7 @@ type PauseActorResponse struct { func (x *PauseActorResponse) Reset() { *x = PauseActorResponse{} - mi := &file_ateapi_proto_msgTypes[58] + mi := &file_ateapi_proto_msgTypes[59] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4483,7 +4540,7 @@ func (x *PauseActorResponse) String() string { func (*PauseActorResponse) ProtoMessage() {} func (x *PauseActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[58] + mi := &file_ateapi_proto_msgTypes[59] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4496,7 +4553,7 @@ func (x *PauseActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PauseActorResponse.ProtoReflect.Descriptor instead. func (*PauseActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{58} + return file_ateapi_proto_rawDescGZIP(), []int{59} } func (x *PauseActorResponse) GetActor() *Actor { @@ -4514,14 +4571,24 @@ type ResumeActorRequest struct { // If true, skip golden snapshot and boot the workload from scratch. // // +k8s:optional - Boot bool `protobuf:"varint,2,opt,name=boot,proto3" json:"boot,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Boot bool `protobuf:"varint,2,opt,name=boot,proto3" json:"boot,omitempty"` + // Reattach to an already-running actor only with its current runtime lease. + // + // +k8s:optional + Lease *ActorLease `protobuf:"bytes,3,opt,name=lease,proto3" json:"lease,omitempty"` + // Claim the actor's runtime lease for an executor-owned workload. The + // legacy router path leaves this false so it can keep resolving already + // running actors without becoming their liveness owner. + // + // +k8s:optional + ClaimRuntimeLease bool `protobuf:"varint,4,opt,name=claim_runtime_lease,json=claimRuntimeLease,proto3" json:"claim_runtime_lease,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResumeActorRequest) Reset() { *x = ResumeActorRequest{} - mi := &file_ateapi_proto_msgTypes[59] + mi := &file_ateapi_proto_msgTypes[60] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4533,7 +4600,7 @@ func (x *ResumeActorRequest) String() string { func (*ResumeActorRequest) ProtoMessage() {} func (x *ResumeActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[59] + mi := &file_ateapi_proto_msgTypes[60] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4546,7 +4613,7 @@ func (x *ResumeActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ResumeActorRequest.ProtoReflect.Descriptor instead. func (*ResumeActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{59} + return file_ateapi_proto_rawDescGZIP(), []int{60} } func (x *ResumeActorRequest) GetActor() *ObjectRef { @@ -4563,19 +4630,35 @@ func (x *ResumeActorRequest) GetBoot() bool { return false } +func (x *ResumeActorRequest) GetLease() *ActorLease { + if x != nil { + return x.Lease + } + return nil +} + +func (x *ResumeActorRequest) GetClaimRuntimeLease() bool { + if x != nil { + return x.ClaimRuntimeLease + } + return false +} + type ResumeActorResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` // True if a resume workflow was executed to activate the actor. // False if the actor was already RUNNING. - Resumed bool `protobuf:"varint,2,opt,name=resumed,proto3" json:"resumed,omitempty"` + Resumed bool `protobuf:"varint,2,opt,name=resumed,proto3" json:"resumed,omitempty"` + // The runtime lease for the running actor. + Lease *ActorLease `protobuf:"bytes,3,opt,name=lease,proto3" json:"lease,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *ResumeActorResponse) Reset() { *x = ResumeActorResponse{} - mi := &file_ateapi_proto_msgTypes[60] + mi := &file_ateapi_proto_msgTypes[61] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4587,7 +4670,7 @@ func (x *ResumeActorResponse) String() string { func (*ResumeActorResponse) ProtoMessage() {} func (x *ResumeActorResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[60] + mi := &file_ateapi_proto_msgTypes[61] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4600,7 +4683,7 @@ func (x *ResumeActorResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ResumeActorResponse.ProtoReflect.Descriptor instead. func (*ResumeActorResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{60} + return file_ateapi_proto_rawDescGZIP(), []int{61} } func (x *ResumeActorResponse) GetActor() *Actor { @@ -4617,6 +4700,178 @@ func (x *ResumeActorResponse) GetResumed() bool { return false } +func (x *ResumeActorResponse) GetLease() *ActorLease { + if x != nil { + return x.Lease + } + return nil +} + +// ActorLease identifies one actor workload incarnation. The token is opaque to +// callers; generation prevents an old token from being reused after reclaim. +type ActorLease struct { + state protoimpl.MessageState `protogen:"open.v1"` + // +k8s:required + Token string `protobuf:"bytes,1,opt,name=token,proto3" json:"token,omitempty"` + // +k8s:required + // +k8s:minimum=1 + Generation int64 `protobuf:"varint,2,opt,name=generation,proto3" json:"generation,omitempty"` + // Server-assigned expiration time. + ExpiresAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorLease) Reset() { + *x = ActorLease{} + mi := &file_ateapi_proto_msgTypes[62] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorLease) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorLease) ProtoMessage() {} + +func (x *ActorLease) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[62] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorLease.ProtoReflect.Descriptor instead. +func (*ActorLease) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{62} +} + +func (x *ActorLease) GetToken() string { + if x != nil { + return x.Token + } + return "" +} + +func (x *ActorLease) GetGeneration() int64 { + if x != nil { + return x.Generation + } + return 0 +} + +func (x *ActorLease) GetExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.ExpiresAt + } + return nil +} + +type RenewActorLeaseRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // +k8s:required + // +k8s:subfield(atespace)=+k8s:required + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + // +k8s:required + Lease *ActorLease `protobuf:"bytes,2,opt,name=lease,proto3" json:"lease,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RenewActorLeaseRequest) Reset() { + *x = RenewActorLeaseRequest{} + mi := &file_ateapi_proto_msgTypes[63] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RenewActorLeaseRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RenewActorLeaseRequest) ProtoMessage() {} + +func (x *RenewActorLeaseRequest) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[63] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RenewActorLeaseRequest.ProtoReflect.Descriptor instead. +func (*RenewActorLeaseRequest) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{63} +} + +func (x *RenewActorLeaseRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +func (x *RenewActorLeaseRequest) GetLease() *ActorLease { + if x != nil { + return x.Lease + } + return nil +} + +type RenewActorLeaseResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Lease *ActorLease `protobuf:"bytes,1,opt,name=lease,proto3" json:"lease,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RenewActorLeaseResponse) Reset() { + *x = RenewActorLeaseResponse{} + mi := &file_ateapi_proto_msgTypes[64] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RenewActorLeaseResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RenewActorLeaseResponse) ProtoMessage() {} + +func (x *RenewActorLeaseResponse) ProtoReflect() protoreflect.Message { + mi := &file_ateapi_proto_msgTypes[64] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RenewActorLeaseResponse.ProtoReflect.Descriptor instead. +func (*RenewActorLeaseResponse) Descriptor() ([]byte, []int) { + return file_ateapi_proto_rawDescGZIP(), []int{64} +} + +func (x *RenewActorLeaseResponse) GetLease() *ActorLease { + if x != nil { + return x.Lease + } + return nil +} + type DeleteActorRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // +k8s:required @@ -4632,7 +4887,7 @@ type DeleteActorRequest struct { func (x *DeleteActorRequest) Reset() { *x = DeleteActorRequest{} - mi := &file_ateapi_proto_msgTypes[61] + mi := &file_ateapi_proto_msgTypes[65] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4644,7 +4899,7 @@ func (x *DeleteActorRequest) String() string { func (*DeleteActorRequest) ProtoMessage() {} func (x *DeleteActorRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[61] + mi := &file_ateapi_proto_msgTypes[65] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4657,7 +4912,7 @@ func (x *DeleteActorRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteActorRequest.ProtoReflect.Descriptor instead. func (*DeleteActorRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{61} + return file_ateapi_proto_rawDescGZIP(), []int{65} } func (x *DeleteActorRequest) GetActor() *ObjectRef { @@ -4688,7 +4943,7 @@ type GetActorEgressPolicyRequest struct { func (x *GetActorEgressPolicyRequest) Reset() { *x = GetActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[62] + mi := &file_ateapi_proto_msgTypes[66] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4700,7 +4955,7 @@ func (x *GetActorEgressPolicyRequest) String() string { func (*GetActorEgressPolicyRequest) ProtoMessage() {} func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[62] + mi := &file_ateapi_proto_msgTypes[66] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4713,7 +4968,7 @@ func (x *GetActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*GetActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{62} + return file_ateapi_proto_rawDescGZIP(), []int{66} } func (x *GetActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -4744,7 +4999,7 @@ type CreateActorEgressPolicyRequest struct { func (x *CreateActorEgressPolicyRequest) Reset() { *x = CreateActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[63] + mi := &file_ateapi_proto_msgTypes[67] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4756,7 +5011,7 @@ func (x *CreateActorEgressPolicyRequest) String() string { func (*CreateActorEgressPolicyRequest) ProtoMessage() {} func (x *CreateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[63] + mi := &file_ateapi_proto_msgTypes[67] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4769,7 +5024,7 @@ func (x *CreateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*CreateActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{63} + return file_ateapi_proto_rawDescGZIP(), []int{67} } func (x *CreateActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -4807,7 +5062,7 @@ type UpdateActorEgressPolicyRequest struct { func (x *UpdateActorEgressPolicyRequest) Reset() { *x = UpdateActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[64] + mi := &file_ateapi_proto_msgTypes[68] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4819,7 +5074,7 @@ func (x *UpdateActorEgressPolicyRequest) String() string { func (*UpdateActorEgressPolicyRequest) ProtoMessage() {} func (x *UpdateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[64] + mi := &file_ateapi_proto_msgTypes[68] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4832,7 +5087,7 @@ func (x *UpdateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*UpdateActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{64} + return file_ateapi_proto_rawDescGZIP(), []int{68} } func (x *UpdateActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -4863,7 +5118,7 @@ type DeleteActorEgressPolicyRequest struct { func (x *DeleteActorEgressPolicyRequest) Reset() { *x = DeleteActorEgressPolicyRequest{} - mi := &file_ateapi_proto_msgTypes[65] + mi := &file_ateapi_proto_msgTypes[69] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4875,7 +5130,7 @@ func (x *DeleteActorEgressPolicyRequest) String() string { func (*DeleteActorEgressPolicyRequest) ProtoMessage() {} func (x *DeleteActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[65] + mi := &file_ateapi_proto_msgTypes[69] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4888,7 +5143,7 @@ func (x *DeleteActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteActorEgressPolicyRequest.ProtoReflect.Descriptor instead. func (*DeleteActorEgressPolicyRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{65} + return file_ateapi_proto_rawDescGZIP(), []int{69} } func (x *DeleteActorEgressPolicyRequest) GetActor() *ObjectRef { @@ -4909,7 +5164,7 @@ type GetTagRequest struct { func (x *GetTagRequest) Reset() { *x = GetTagRequest{} - mi := &file_ateapi_proto_msgTypes[66] + mi := &file_ateapi_proto_msgTypes[70] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4921,7 +5176,7 @@ func (x *GetTagRequest) String() string { func (*GetTagRequest) ProtoMessage() {} func (x *GetTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[66] + mi := &file_ateapi_proto_msgTypes[70] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4934,7 +5189,7 @@ func (x *GetTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetTagRequest.ProtoReflect.Descriptor instead. func (*GetTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{66} + return file_ateapi_proto_rawDescGZIP(), []int{70} } func (x *GetTagRequest) GetTag() *ObjectRef { @@ -4970,7 +5225,7 @@ type ListTagsRequest struct { func (x *ListTagsRequest) Reset() { *x = ListTagsRequest{} - mi := &file_ateapi_proto_msgTypes[67] + mi := &file_ateapi_proto_msgTypes[71] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -4982,7 +5237,7 @@ func (x *ListTagsRequest) String() string { func (*ListTagsRequest) ProtoMessage() {} func (x *ListTagsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[67] + mi := &file_ateapi_proto_msgTypes[71] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -4995,7 +5250,7 @@ func (x *ListTagsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListTagsRequest.ProtoReflect.Descriptor instead. func (*ListTagsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{67} + return file_ateapi_proto_rawDescGZIP(), []int{71} } func (x *ListTagsRequest) GetAtespace() string { @@ -5029,7 +5284,7 @@ type ListTagsResponse struct { func (x *ListTagsResponse) Reset() { *x = ListTagsResponse{} - mi := &file_ateapi_proto_msgTypes[68] + mi := &file_ateapi_proto_msgTypes[72] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5041,7 +5296,7 @@ func (x *ListTagsResponse) String() string { func (*ListTagsResponse) ProtoMessage() {} func (x *ListTagsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[68] + mi := &file_ateapi_proto_msgTypes[72] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5054,7 +5309,7 @@ func (x *ListTagsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListTagsResponse.ProtoReflect.Descriptor instead. func (*ListTagsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{68} + return file_ateapi_proto_rawDescGZIP(), []int{72} } func (x *ListTagsResponse) GetTags() []*Tag { @@ -5099,7 +5354,7 @@ type CreateTagRequest struct { func (x *CreateTagRequest) Reset() { *x = CreateTagRequest{} - mi := &file_ateapi_proto_msgTypes[69] + mi := &file_ateapi_proto_msgTypes[73] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5111,7 +5366,7 @@ func (x *CreateTagRequest) String() string { func (*CreateTagRequest) ProtoMessage() {} func (x *CreateTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[69] + mi := &file_ateapi_proto_msgTypes[73] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5124,7 +5379,7 @@ func (x *CreateTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateTagRequest.ProtoReflect.Descriptor instead. func (*CreateTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{69} + return file_ateapi_proto_rawDescGZIP(), []int{73} } func (x *CreateTagRequest) GetTag() *Tag { @@ -5154,7 +5409,7 @@ type UpdateTagRequest struct { func (x *UpdateTagRequest) Reset() { *x = UpdateTagRequest{} - mi := &file_ateapi_proto_msgTypes[70] + mi := &file_ateapi_proto_msgTypes[74] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5166,7 +5421,7 @@ func (x *UpdateTagRequest) String() string { func (*UpdateTagRequest) ProtoMessage() {} func (x *UpdateTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[70] + mi := &file_ateapi_proto_msgTypes[74] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5179,7 +5434,7 @@ func (x *UpdateTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateTagRequest.ProtoReflect.Descriptor instead. func (*UpdateTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{70} + return file_ateapi_proto_rawDescGZIP(), []int{74} } func (x *UpdateTagRequest) GetTag() *Tag { @@ -5200,7 +5455,7 @@ type DeleteTagRequest struct { func (x *DeleteTagRequest) Reset() { *x = DeleteTagRequest{} - mi := &file_ateapi_proto_msgTypes[71] + mi := &file_ateapi_proto_msgTypes[75] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5212,7 +5467,7 @@ func (x *DeleteTagRequest) String() string { func (*DeleteTagRequest) ProtoMessage() {} func (x *DeleteTagRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[71] + mi := &file_ateapi_proto_msgTypes[75] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5225,7 +5480,7 @@ func (x *DeleteTagRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteTagRequest.ProtoReflect.Descriptor instead. func (*DeleteTagRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{71} + return file_ateapi_proto_rawDescGZIP(), []int{75} } func (x *DeleteTagRequest) GetTag() *ObjectRef { @@ -5261,7 +5516,7 @@ type DeleteOptions struct { func (x *DeleteOptions) Reset() { *x = DeleteOptions{} - mi := &file_ateapi_proto_msgTypes[72] + mi := &file_ateapi_proto_msgTypes[76] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5273,7 +5528,7 @@ func (x *DeleteOptions) String() string { func (*DeleteOptions) ProtoMessage() {} func (x *DeleteOptions) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[72] + mi := &file_ateapi_proto_msgTypes[76] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5286,7 +5541,7 @@ func (x *DeleteOptions) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteOptions.ProtoReflect.Descriptor instead. func (*DeleteOptions) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{72} + return file_ateapi_proto_rawDescGZIP(), []int{76} } func (x *DeleteOptions) GetVersion() int64 { @@ -5330,7 +5585,7 @@ type ListWorkerActorAssignmentsRequest struct { func (x *ListWorkerActorAssignmentsRequest) Reset() { *x = ListWorkerActorAssignmentsRequest{} - mi := &file_ateapi_proto_msgTypes[73] + mi := &file_ateapi_proto_msgTypes[77] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5342,7 +5597,7 @@ func (x *ListWorkerActorAssignmentsRequest) String() string { func (*ListWorkerActorAssignmentsRequest) ProtoMessage() {} func (x *ListWorkerActorAssignmentsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[73] + mi := &file_ateapi_proto_msgTypes[77] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5355,7 +5610,7 @@ func (x *ListWorkerActorAssignmentsRequest) ProtoReflect() protoreflect.Message // Deprecated: Use ListWorkerActorAssignmentsRequest.ProtoReflect.Descriptor instead. func (*ListWorkerActorAssignmentsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{73} + return file_ateapi_proto_rawDescGZIP(), []int{77} } func (x *ListWorkerActorAssignmentsRequest) GetWorker() *ObjectRef { @@ -5392,7 +5647,7 @@ type ListWorkerActorAssignmentsResponse struct { func (x *ListWorkerActorAssignmentsResponse) Reset() { *x = ListWorkerActorAssignmentsResponse{} - mi := &file_ateapi_proto_msgTypes[74] + mi := &file_ateapi_proto_msgTypes[78] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5404,7 +5659,7 @@ func (x *ListWorkerActorAssignmentsResponse) String() string { func (*ListWorkerActorAssignmentsResponse) ProtoMessage() {} func (x *ListWorkerActorAssignmentsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[74] + mi := &file_ateapi_proto_msgTypes[78] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5417,7 +5672,7 @@ func (x *ListWorkerActorAssignmentsResponse) ProtoReflect() protoreflect.Message // Deprecated: Use ListWorkerActorAssignmentsResponse.ProtoReflect.Descriptor instead. func (*ListWorkerActorAssignmentsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{74} + return file_ateapi_proto_rawDescGZIP(), []int{78} } func (x *ListWorkerActorAssignmentsResponse) GetActorAssignments() []*ActorAssignment { @@ -5455,7 +5710,7 @@ type ListWorkersRequest struct { func (x *ListWorkersRequest) Reset() { *x = ListWorkersRequest{} - mi := &file_ateapi_proto_msgTypes[75] + mi := &file_ateapi_proto_msgTypes[79] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5467,7 +5722,7 @@ func (x *ListWorkersRequest) String() string { func (*ListWorkersRequest) ProtoMessage() {} func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[75] + mi := &file_ateapi_proto_msgTypes[79] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5480,7 +5735,7 @@ func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkersRequest.ProtoReflect.Descriptor instead. func (*ListWorkersRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{75} + return file_ateapi_proto_rawDescGZIP(), []int{79} } func (x *ListWorkersRequest) GetPageSize() int32 { @@ -5509,7 +5764,7 @@ type ListWorkersResponse struct { func (x *ListWorkersResponse) Reset() { *x = ListWorkersResponse{} - mi := &file_ateapi_proto_msgTypes[76] + mi := &file_ateapi_proto_msgTypes[80] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5521,7 +5776,7 @@ func (x *ListWorkersResponse) String() string { func (*ListWorkersResponse) ProtoMessage() {} func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[76] + mi := &file_ateapi_proto_msgTypes[80] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5534,7 +5789,7 @@ func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkersResponse.ProtoReflect.Descriptor instead. func (*ListWorkersResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{76} + return file_ateapi_proto_rawDescGZIP(), []int{80} } func (x *ListWorkersResponse) GetWorkers() []*Worker { @@ -5564,7 +5819,7 @@ type GetWorkerRequest struct { func (x *GetWorkerRequest) Reset() { *x = GetWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[77] + mi := &file_ateapi_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5576,7 +5831,7 @@ func (x *GetWorkerRequest) String() string { func (*GetWorkerRequest) ProtoMessage() {} func (x *GetWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[77] + mi := &file_ateapi_proto_msgTypes[81] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5589,7 +5844,7 @@ func (x *GetWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkerRequest.ProtoReflect.Descriptor instead. func (*GetWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{77} + return file_ateapi_proto_rawDescGZIP(), []int{81} } func (x *GetWorkerRequest) GetWorker() *ObjectRef { @@ -5611,7 +5866,7 @@ type CreateWorkerRequest struct { func (x *CreateWorkerRequest) Reset() { *x = CreateWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[78] + mi := &file_ateapi_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5623,7 +5878,7 @@ func (x *CreateWorkerRequest) String() string { func (*CreateWorkerRequest) ProtoMessage() {} func (x *CreateWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[78] + mi := &file_ateapi_proto_msgTypes[82] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5636,7 +5891,7 @@ func (x *CreateWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CreateWorkerRequest.ProtoReflect.Descriptor instead. func (*CreateWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{78} + return file_ateapi_proto_rawDescGZIP(), []int{82} } func (x *CreateWorkerRequest) GetWorker() *Worker { @@ -5671,7 +5926,7 @@ type UpdateWorkerRequest struct { func (x *UpdateWorkerRequest) Reset() { *x = UpdateWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[79] + mi := &file_ateapi_proto_msgTypes[83] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5683,7 +5938,7 @@ func (x *UpdateWorkerRequest) String() string { func (*UpdateWorkerRequest) ProtoMessage() {} func (x *UpdateWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[79] + mi := &file_ateapi_proto_msgTypes[83] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5696,7 +5951,7 @@ func (x *UpdateWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use UpdateWorkerRequest.ProtoReflect.Descriptor instead. func (*UpdateWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{79} + return file_ateapi_proto_rawDescGZIP(), []int{83} } func (x *UpdateWorkerRequest) GetWorker() *Worker { @@ -5723,7 +5978,7 @@ type DeleteWorkerRequest struct { func (x *DeleteWorkerRequest) Reset() { *x = DeleteWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[80] + mi := &file_ateapi_proto_msgTypes[84] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5735,7 +5990,7 @@ func (x *DeleteWorkerRequest) String() string { func (*DeleteWorkerRequest) ProtoMessage() {} func (x *DeleteWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[80] + mi := &file_ateapi_proto_msgTypes[84] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5748,7 +6003,7 @@ func (x *DeleteWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DeleteWorkerRequest.ProtoReflect.Descriptor instead. func (*DeleteWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{80} + return file_ateapi_proto_rawDescGZIP(), []int{84} } func (x *DeleteWorkerRequest) GetWorker() *ObjectRef { @@ -5778,7 +6033,7 @@ type DrainWorkerRequest struct { func (x *DrainWorkerRequest) Reset() { *x = DrainWorkerRequest{} - mi := &file_ateapi_proto_msgTypes[81] + mi := &file_ateapi_proto_msgTypes[85] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5790,7 +6045,7 @@ func (x *DrainWorkerRequest) String() string { func (*DrainWorkerRequest) ProtoMessage() {} func (x *DrainWorkerRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[81] + mi := &file_ateapi_proto_msgTypes[85] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5803,7 +6058,7 @@ func (x *DrainWorkerRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use DrainWorkerRequest.ProtoReflect.Descriptor instead. func (*DrainWorkerRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{81} + return file_ateapi_proto_rawDescGZIP(), []int{85} } func (x *DrainWorkerRequest) GetWorker() *ObjectRef { @@ -5841,7 +6096,7 @@ type ListActorsRequest struct { func (x *ListActorsRequest) Reset() { *x = ListActorsRequest{} - mi := &file_ateapi_proto_msgTypes[82] + mi := &file_ateapi_proto_msgTypes[86] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5853,7 +6108,7 @@ func (x *ListActorsRequest) String() string { func (*ListActorsRequest) ProtoMessage() {} func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[82] + mi := &file_ateapi_proto_msgTypes[86] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5866,7 +6121,7 @@ func (x *ListActorsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsRequest.ProtoReflect.Descriptor instead. func (*ListActorsRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{82} + return file_ateapi_proto_rawDescGZIP(), []int{86} } func (x *ListActorsRequest) GetAtespace() string { @@ -5902,7 +6157,7 @@ type ListActorsResponse struct { func (x *ListActorsResponse) Reset() { *x = ListActorsResponse{} - mi := &file_ateapi_proto_msgTypes[83] + mi := &file_ateapi_proto_msgTypes[87] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -5914,7 +6169,7 @@ func (x *ListActorsResponse) String() string { func (*ListActorsResponse) ProtoMessage() {} func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[83] + mi := &file_ateapi_proto_msgTypes[87] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -5927,7 +6182,7 @@ func (x *ListActorsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListActorsResponse.ProtoReflect.Descriptor instead. func (*ListActorsResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{83} + return file_ateapi_proto_rawDescGZIP(), []int{87} } func (x *ListActorsResponse) GetActors() []*Actor { @@ -6016,7 +6271,7 @@ type Worker struct { func (x *Worker) Reset() { *x = Worker{} - mi := &file_ateapi_proto_msgTypes[84] + mi := &file_ateapi_proto_msgTypes[88] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6028,7 +6283,7 @@ func (x *Worker) String() string { func (*Worker) ProtoMessage() {} func (x *Worker) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[84] + mi := &file_ateapi_proto_msgTypes[88] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6041,7 +6296,7 @@ func (x *Worker) ProtoReflect() protoreflect.Message { // Deprecated: Use Worker.ProtoReflect.Descriptor instead. func (*Worker) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{84} + return file_ateapi_proto_rawDescGZIP(), []int{88} } func (x *Worker) GetMetadata() *ResourceMetadata { @@ -6143,7 +6398,7 @@ type WorkerStatus struct { func (x *WorkerStatus) Reset() { *x = WorkerStatus{} - mi := &file_ateapi_proto_msgTypes[85] + mi := &file_ateapi_proto_msgTypes[89] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6155,7 +6410,7 @@ func (x *WorkerStatus) String() string { func (*WorkerStatus) ProtoMessage() {} func (x *WorkerStatus) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[85] + mi := &file_ateapi_proto_msgTypes[89] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6168,7 +6423,7 @@ func (x *WorkerStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerStatus.ProtoReflect.Descriptor instead. func (*WorkerStatus) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{85} + return file_ateapi_proto_rawDescGZIP(), []int{89} } func (x *WorkerStatus) GetState() WorkerState { @@ -6214,7 +6469,7 @@ type WorkerResources struct { func (x *WorkerResources) Reset() { *x = WorkerResources{} - mi := &file_ateapi_proto_msgTypes[86] + mi := &file_ateapi_proto_msgTypes[90] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6226,7 +6481,7 @@ func (x *WorkerResources) String() string { func (*WorkerResources) ProtoMessage() {} func (x *WorkerResources) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[86] + mi := &file_ateapi_proto_msgTypes[90] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6239,7 +6494,7 @@ func (x *WorkerResources) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkerResources.ProtoReflect.Descriptor instead. func (*WorkerResources) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{86} + return file_ateapi_proto_rawDescGZIP(), []int{90} } func (x *WorkerResources) GetResources() *Resources { @@ -6293,7 +6548,7 @@ type ActorAssignment struct { func (x *ActorAssignment) Reset() { *x = ActorAssignment{} - mi := &file_ateapi_proto_msgTypes[87] + mi := &file_ateapi_proto_msgTypes[91] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6305,7 +6560,7 @@ func (x *ActorAssignment) String() string { func (*ActorAssignment) ProtoMessage() {} func (x *ActorAssignment) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[87] + mi := &file_ateapi_proto_msgTypes[91] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6318,7 +6573,7 @@ func (x *ActorAssignment) ProtoReflect() protoreflect.Message { // Deprecated: Use ActorAssignment.ProtoReflect.Descriptor instead. func (*ActorAssignment) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{87} + return file_ateapi_proto_rawDescGZIP(), []int{91} } func (x *ActorAssignment) GetMetadata() *ResourceMetadata { @@ -6376,7 +6631,7 @@ type SetWorkerCapacityRequest struct { func (x *SetWorkerCapacityRequest) Reset() { *x = SetWorkerCapacityRequest{} - mi := &file_ateapi_proto_msgTypes[88] + mi := &file_ateapi_proto_msgTypes[92] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6388,7 +6643,7 @@ func (x *SetWorkerCapacityRequest) String() string { func (*SetWorkerCapacityRequest) ProtoMessage() {} func (x *SetWorkerCapacityRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[88] + mi := &file_ateapi_proto_msgTypes[92] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6401,7 +6656,7 @@ func (x *SetWorkerCapacityRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SetWorkerCapacityRequest.ProtoReflect.Descriptor instead. func (*SetWorkerCapacityRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{88} + return file_ateapi_proto_rawDescGZIP(), []int{92} } func (x *SetWorkerCapacityRequest) GetWorker() *ObjectRef { @@ -6428,7 +6683,7 @@ type SetWorkerCapacityResponse struct { func (x *SetWorkerCapacityResponse) Reset() { *x = SetWorkerCapacityResponse{} - mi := &file_ateapi_proto_msgTypes[89] + mi := &file_ateapi_proto_msgTypes[93] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6440,7 +6695,7 @@ func (x *SetWorkerCapacityResponse) String() string { func (*SetWorkerCapacityResponse) ProtoMessage() {} func (x *SetWorkerCapacityResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[89] + mi := &file_ateapi_proto_msgTypes[93] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6453,7 +6708,7 @@ func (x *SetWorkerCapacityResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SetWorkerCapacityResponse.ProtoReflect.Descriptor instead. func (*SetWorkerCapacityResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{89} + return file_ateapi_proto_rawDescGZIP(), []int{93} } func (x *SetWorkerCapacityResponse) GetWorker() *Worker { @@ -6488,7 +6743,7 @@ type MintJWTRequest struct { func (x *MintJWTRequest) Reset() { *x = MintJWTRequest{} - mi := &file_ateapi_proto_msgTypes[90] + mi := &file_ateapi_proto_msgTypes[94] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6500,7 +6755,7 @@ func (x *MintJWTRequest) String() string { func (*MintJWTRequest) ProtoMessage() {} func (x *MintJWTRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[90] + mi := &file_ateapi_proto_msgTypes[94] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6513,7 +6768,7 @@ func (x *MintJWTRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintJWTRequest.ProtoReflect.Descriptor instead. func (*MintJWTRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{90} + return file_ateapi_proto_rawDescGZIP(), []int{94} } func (x *MintJWTRequest) GetAudience() []string { @@ -6572,7 +6827,7 @@ type MintJWTResponse struct { func (x *MintJWTResponse) Reset() { *x = MintJWTResponse{} - mi := &file_ateapi_proto_msgTypes[91] + mi := &file_ateapi_proto_msgTypes[95] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6584,7 +6839,7 @@ func (x *MintJWTResponse) String() string { func (*MintJWTResponse) ProtoMessage() {} func (x *MintJWTResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[91] + mi := &file_ateapi_proto_msgTypes[95] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6597,7 +6852,7 @@ func (x *MintJWTResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintJWTResponse.ProtoReflect.Descriptor instead. func (*MintJWTResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{91} + return file_ateapi_proto_rawDescGZIP(), []int{95} } func (x *MintJWTResponse) GetActorJwt() string { @@ -6644,7 +6899,7 @@ type MintCertRequest struct { func (x *MintCertRequest) Reset() { *x = MintCertRequest{} - mi := &file_ateapi_proto_msgTypes[92] + mi := &file_ateapi_proto_msgTypes[96] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6656,7 +6911,7 @@ func (x *MintCertRequest) String() string { func (*MintCertRequest) ProtoMessage() {} func (x *MintCertRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[92] + mi := &file_ateapi_proto_msgTypes[96] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6669,7 +6924,7 @@ func (x *MintCertRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use MintCertRequest.ProtoReflect.Descriptor instead. func (*MintCertRequest) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{92} + return file_ateapi_proto_rawDescGZIP(), []int{96} } func (x *MintCertRequest) GetWorker() *ObjectRef { @@ -6712,7 +6967,7 @@ type MintCertResponse struct { func (x *MintCertResponse) Reset() { *x = MintCertResponse{} - mi := &file_ateapi_proto_msgTypes[93] + mi := &file_ateapi_proto_msgTypes[97] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -6724,7 +6979,7 @@ func (x *MintCertResponse) String() string { func (*MintCertResponse) ProtoMessage() {} func (x *MintCertResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateapi_proto_msgTypes[93] + mi := &file_ateapi_proto_msgTypes[97] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -6737,7 +6992,7 @@ func (x *MintCertResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MintCertResponse.ProtoReflect.Descriptor instead. func (*MintCertResponse) Descriptor() ([]byte, []int) { - return file_ateapi_proto_rawDescGZIP(), []int{93} + return file_ateapi_proto_rawDescGZIP(), []int{97} } func (x *MintCertResponse) GetActorCertificates() [][]byte { @@ -6972,17 +7227,36 @@ const file_ateapi_proto_rawDesc = "" + "\x13SuspendActorRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\";\n" + "\x14SuspendActorResponse\x12#\n" + - "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\"<\n" + + "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\"q\n" + + "\x1cSuspendActorWithLeaseRequest\x12'\n" + + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12(\n" + + "\x05lease\x18\x02 \x01(\v2\x12.ateapi.ActorLeaseR\x05lease\"<\n" + "\x11PauseActorRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\"9\n" + "\x12PauseActorResponse\x12#\n" + - "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\"Q\n" + + "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\"\xab\x01\n" + "\x12ResumeActorRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x12\n" + - "\x04boot\x18\x02 \x01(\bR\x04boot\"T\n" + + "\x04boot\x18\x02 \x01(\bR\x04boot\x12(\n" + + "\x05lease\x18\x03 \x01(\v2\x12.ateapi.ActorLeaseR\x05lease\x12.\n" + + "\x13claim_runtime_lease\x18\x04 \x01(\bR\x11claimRuntimeLease\"~\n" + "\x13ResumeActorResponse\x12#\n" + "\x05actor\x18\x01 \x01(\v2\r.ateapi.ActorR\x05actor\x12\x18\n" + - "\aresumed\x18\x02 \x01(\bR\aresumed\"Z\n" + + "\aresumed\x18\x02 \x01(\bR\aresumed\x12(\n" + + "\x05lease\x18\x03 \x01(\v2\x12.ateapi.ActorLeaseR\x05lease\"}\n" + + "\n" + + "ActorLease\x12\x14\n" + + "\x05token\x18\x01 \x01(\tR\x05token\x12\x1e\n" + + "\n" + + "generation\x18\x02 \x01(\x03R\n" + + "generation\x129\n" + + "\n" + + "expires_at\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\texpiresAt\"k\n" + + "\x16RenewActorLeaseRequest\x12'\n" + + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12(\n" + + "\x05lease\x18\x02 \x01(\v2\x12.ateapi.ActorLeaseR\x05lease\"C\n" + + "\x17RenewActorLeaseResponse\x12(\n" + + "\x05lease\x18\x01 \x01(\v2\x12.ateapi.ActorLeaseR\x05lease\"Z\n" + "\x12DeleteActorRequest\x12'\n" + "\x05actor\x18\x01 \x01(\v2\x11.ateapi.ObjectRefR\x05actor\x12\x1b\n" + "\tany_state\x18\x02 \x01(\bR\banyState\"F\n" + @@ -7136,15 +7410,17 @@ const file_ateapi_proto_rawDesc = "" + "\x15WORKER_STATE_DRAINING\x10\x02*k\n" + "\x17ActorCertificatePurpose\x12)\n" + "%ACTOR_CERTIFICATE_PURPOSE_UNSPECIFIED\x10\x00\x12%\n" + - "!ACTOR_CERTIFICATE_PURPOSE_ATUNNEL\x10\x012\xf3\x11\n" + + "!ACTOR_CERTIFICATE_PURPOSE_ATUNNEL\x10\x012\xa8\x13\n" + "\aControl\x124\n" + "\bGetActor\x12\x17.ateapi.GetActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n" + "\vCreateActor\x12\x1a.ateapi.CreateActorRequest\x1a\r.ateapi.Actor\"\x00\x12:\n" + "\vUpdateActor\x12\x1a.ateapi.UpdateActorRequest\x1a\r.ateapi.Actor\"\x00\x12K\n" + - "\fSuspendActor\x12\x1b.ateapi.SuspendActorRequest\x1a\x1c.ateapi.SuspendActorResponse\"\x00\x12E\n" + + "\fSuspendActor\x12\x1b.ateapi.SuspendActorRequest\x1a\x1c.ateapi.SuspendActorResponse\"\x00\x12]\n" + + "\x15SuspendActorWithLease\x12$.ateapi.SuspendActorWithLeaseRequest\x1a\x1c.ateapi.SuspendActorResponse\"\x00\x12E\n" + "\n" + "PauseActor\x12\x19.ateapi.PauseActorRequest\x1a\x1a.ateapi.PauseActorResponse\"\x00\x12H\n" + - "\vResumeActor\x12\x1a.ateapi.ResumeActorRequest\x1a\x1b.ateapi.ResumeActorResponse\"\x00\x12:\n" + + "\vResumeActor\x12\x1a.ateapi.ResumeActorRequest\x1a\x1b.ateapi.ResumeActorResponse\"\x00\x12T\n" + + "\x0fRenewActorLease\x12\x1e.ateapi.RenewActorLeaseRequest\x1a\x1f.ateapi.RenewActorLeaseResponse\"\x00\x12:\n" + "\vDeleteActor\x12\x1a.ateapi.DeleteActorRequest\x1a\r.ateapi.Actor\"\x00\x12S\n" + "\x14GetActorEgressPolicy\x12#.ateapi.GetActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n" + "\x17CreateActorEgressPolicy\x12&.ateapi.CreateActorEgressPolicyRequest\x1a\x14.ateapi.EgressPolicy\"\x00\x12Y\n" + @@ -7191,7 +7467,7 @@ func file_ateapi_proto_rawDescGZIP() []byte { } var file_ateapi_proto_enumTypes = make([]protoimpl.EnumInfo, 9) -var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 97) +var file_ateapi_proto_msgTypes = make([]protoimpl.MessageInfo, 101) var file_ateapi_proto_goTypes = []any{ (SnapshotContentScope)(0), // 0: ateapi.SnapshotContentScope (TagScope)(0), // 1: ateapi.TagScope @@ -7259,57 +7535,61 @@ var file_ateapi_proto_goTypes = []any{ (*UpdateActorRequest)(nil), // 63: ateapi.UpdateActorRequest (*SuspendActorRequest)(nil), // 64: ateapi.SuspendActorRequest (*SuspendActorResponse)(nil), // 65: ateapi.SuspendActorResponse - (*PauseActorRequest)(nil), // 66: ateapi.PauseActorRequest - (*PauseActorResponse)(nil), // 67: ateapi.PauseActorResponse - (*ResumeActorRequest)(nil), // 68: ateapi.ResumeActorRequest - (*ResumeActorResponse)(nil), // 69: ateapi.ResumeActorResponse - (*DeleteActorRequest)(nil), // 70: ateapi.DeleteActorRequest - (*GetActorEgressPolicyRequest)(nil), // 71: ateapi.GetActorEgressPolicyRequest - (*CreateActorEgressPolicyRequest)(nil), // 72: ateapi.CreateActorEgressPolicyRequest - (*UpdateActorEgressPolicyRequest)(nil), // 73: ateapi.UpdateActorEgressPolicyRequest - (*DeleteActorEgressPolicyRequest)(nil), // 74: ateapi.DeleteActorEgressPolicyRequest - (*GetTagRequest)(nil), // 75: ateapi.GetTagRequest - (*ListTagsRequest)(nil), // 76: ateapi.ListTagsRequest - (*ListTagsResponse)(nil), // 77: ateapi.ListTagsResponse - (*CreateTagRequest)(nil), // 78: ateapi.CreateTagRequest - (*UpdateTagRequest)(nil), // 79: ateapi.UpdateTagRequest - (*DeleteTagRequest)(nil), // 80: ateapi.DeleteTagRequest - (*DeleteOptions)(nil), // 81: ateapi.DeleteOptions - (*ListWorkerActorAssignmentsRequest)(nil), // 82: ateapi.ListWorkerActorAssignmentsRequest - (*ListWorkerActorAssignmentsResponse)(nil), // 83: ateapi.ListWorkerActorAssignmentsResponse - (*ListWorkersRequest)(nil), // 84: ateapi.ListWorkersRequest - (*ListWorkersResponse)(nil), // 85: ateapi.ListWorkersResponse - (*GetWorkerRequest)(nil), // 86: ateapi.GetWorkerRequest - (*CreateWorkerRequest)(nil), // 87: ateapi.CreateWorkerRequest - (*UpdateWorkerRequest)(nil), // 88: ateapi.UpdateWorkerRequest - (*DeleteWorkerRequest)(nil), // 89: ateapi.DeleteWorkerRequest - (*DrainWorkerRequest)(nil), // 90: ateapi.DrainWorkerRequest - (*ListActorsRequest)(nil), // 91: ateapi.ListActorsRequest - (*ListActorsResponse)(nil), // 92: ateapi.ListActorsResponse - (*Worker)(nil), // 93: ateapi.Worker - (*WorkerStatus)(nil), // 94: ateapi.WorkerStatus - (*WorkerResources)(nil), // 95: ateapi.WorkerResources - (*ActorAssignment)(nil), // 96: ateapi.ActorAssignment - (*SetWorkerCapacityRequest)(nil), // 97: ateapi.SetWorkerCapacityRequest - (*SetWorkerCapacityResponse)(nil), // 98: ateapi.SetWorkerCapacityResponse - (*MintJWTRequest)(nil), // 99: ateapi.MintJWTRequest - (*MintJWTResponse)(nil), // 100: ateapi.MintJWTResponse - (*MintCertRequest)(nil), // 101: ateapi.MintCertRequest - (*MintCertResponse)(nil), // 102: ateapi.MintCertResponse - nil, // 103: ateapi.Selector.MatchLabelsEntry - nil, // 104: ateapi.ExternalVolume.VolumeContextEntry - nil, // 105: ateapi.Worker.LabelsEntry - (*timestamppb.Timestamp)(nil), // 106: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 107: google.protobuf.Empty + (*SuspendActorWithLeaseRequest)(nil), // 66: ateapi.SuspendActorWithLeaseRequest + (*PauseActorRequest)(nil), // 67: ateapi.PauseActorRequest + (*PauseActorResponse)(nil), // 68: ateapi.PauseActorResponse + (*ResumeActorRequest)(nil), // 69: ateapi.ResumeActorRequest + (*ResumeActorResponse)(nil), // 70: ateapi.ResumeActorResponse + (*ActorLease)(nil), // 71: ateapi.ActorLease + (*RenewActorLeaseRequest)(nil), // 72: ateapi.RenewActorLeaseRequest + (*RenewActorLeaseResponse)(nil), // 73: ateapi.RenewActorLeaseResponse + (*DeleteActorRequest)(nil), // 74: ateapi.DeleteActorRequest + (*GetActorEgressPolicyRequest)(nil), // 75: ateapi.GetActorEgressPolicyRequest + (*CreateActorEgressPolicyRequest)(nil), // 76: ateapi.CreateActorEgressPolicyRequest + (*UpdateActorEgressPolicyRequest)(nil), // 77: ateapi.UpdateActorEgressPolicyRequest + (*DeleteActorEgressPolicyRequest)(nil), // 78: ateapi.DeleteActorEgressPolicyRequest + (*GetTagRequest)(nil), // 79: ateapi.GetTagRequest + (*ListTagsRequest)(nil), // 80: ateapi.ListTagsRequest + (*ListTagsResponse)(nil), // 81: ateapi.ListTagsResponse + (*CreateTagRequest)(nil), // 82: ateapi.CreateTagRequest + (*UpdateTagRequest)(nil), // 83: ateapi.UpdateTagRequest + (*DeleteTagRequest)(nil), // 84: ateapi.DeleteTagRequest + (*DeleteOptions)(nil), // 85: ateapi.DeleteOptions + (*ListWorkerActorAssignmentsRequest)(nil), // 86: ateapi.ListWorkerActorAssignmentsRequest + (*ListWorkerActorAssignmentsResponse)(nil), // 87: ateapi.ListWorkerActorAssignmentsResponse + (*ListWorkersRequest)(nil), // 88: ateapi.ListWorkersRequest + (*ListWorkersResponse)(nil), // 89: ateapi.ListWorkersResponse + (*GetWorkerRequest)(nil), // 90: ateapi.GetWorkerRequest + (*CreateWorkerRequest)(nil), // 91: ateapi.CreateWorkerRequest + (*UpdateWorkerRequest)(nil), // 92: ateapi.UpdateWorkerRequest + (*DeleteWorkerRequest)(nil), // 93: ateapi.DeleteWorkerRequest + (*DrainWorkerRequest)(nil), // 94: ateapi.DrainWorkerRequest + (*ListActorsRequest)(nil), // 95: ateapi.ListActorsRequest + (*ListActorsResponse)(nil), // 96: ateapi.ListActorsResponse + (*Worker)(nil), // 97: ateapi.Worker + (*WorkerStatus)(nil), // 98: ateapi.WorkerStatus + (*WorkerResources)(nil), // 99: ateapi.WorkerResources + (*ActorAssignment)(nil), // 100: ateapi.ActorAssignment + (*SetWorkerCapacityRequest)(nil), // 101: ateapi.SetWorkerCapacityRequest + (*SetWorkerCapacityResponse)(nil), // 102: ateapi.SetWorkerCapacityResponse + (*MintJWTRequest)(nil), // 103: ateapi.MintJWTRequest + (*MintJWTResponse)(nil), // 104: ateapi.MintJWTResponse + (*MintCertRequest)(nil), // 105: ateapi.MintCertRequest + (*MintCertResponse)(nil), // 106: ateapi.MintCertResponse + nil, // 107: ateapi.Selector.MatchLabelsEntry + nil, // 108: ateapi.ExternalVolume.VolumeContextEntry + nil, // 109: ateapi.Worker.LabelsEntry + (*timestamppb.Timestamp)(nil), // 110: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 111: google.protobuf.Empty } var file_ateapi_proto_depIdxs = []int32{ 0, // 0: ateapi.ExternalSnapshot.content_scope:type_name -> ateapi.SnapshotContentScope 0, // 1: ateapi.LocalSnapshotInfo.content_scope:type_name -> ateapi.SnapshotContentScope - 103, // 2: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry - 106, // 3: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp - 106, // 4: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 107, // 2: ateapi.Selector.match_labels:type_name -> ateapi.Selector.MatchLabelsEntry + 110, // 3: ateapi.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 110, // 4: ateapi.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp 8, // 5: ateapi.ExternalVolume.status:type_name -> ateapi.ExternalVolume.Status - 104, // 6: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry + 108, // 6: ateapi.ExternalVolume.volume_context:type_name -> ateapi.ExternalVolume.VolumeContextEntry 12, // 7: ateapi.Actor.metadata:type_name -> ateapi.ResourceMetadata 26, // 8: ateapi.Actor.actor_template:type_name -> ateapi.ObjectRef 11, // 9: ateapi.Actor.worker_selector:type_name -> ateapi.Selector @@ -7319,7 +7599,7 @@ var file_ateapi_proto_depIdxs = []int32{ 16, // 13: ateapi.EgressPolicy.rules:type_name -> ateapi.EgressRule 17, // 14: ateapi.EgressRule.hostnames:type_name -> ateapi.HostnameRule 18, // 15: ateapi.EgressRule.ip_blocks:type_name -> ateapi.IPBlockRule - 107, // 16: ateapi.EgressRule.all:type_name -> google.protobuf.Empty + 111, // 16: ateapi.EgressRule.all:type_name -> google.protobuf.Empty 19, // 17: ateapi.HostnameRule.effects:type_name -> ateapi.EgressRuleEffects 20, // 18: ateapi.EgressRuleEffects.inject_static_headers:type_name -> ateapi.CredentialHeaderInjection 2, // 19: ateapi.ActorStatus.state:type_name -> ateapi.ActorState @@ -7344,7 +7624,7 @@ var file_ateapi_proto_depIdxs = []int32{ 31, // 38: ateapi.ActorTemplate.status:type_name -> ateapi.ActorTemplateStatus 29, // 39: ateapi.Resources.limits:type_name -> ateapi.Limits 9, // 40: ateapi.GoldenSnapshotStatus.golden_snapshot:type_name -> ateapi.ExternalSnapshot - 106, // 41: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp + 110, // 41: ateapi.GoldenSnapshotStatus.take_golden_snapshot_at:type_name -> google.protobuf.Timestamp 30, // 42: ateapi.ActorTemplateStatus.golden_snapshot_status:type_name -> ateapi.GoldenSnapshotStatus 3, // 43: ateapi.SandboxConfig.sandbox_class:type_name -> ateapi.SandboxClass 0, // 44: ateapi.SnapshotsConfig.on_pause:type_name -> ateapi.SnapshotContentScope @@ -7380,123 +7660,135 @@ var file_ateapi_proto_depIdxs = []int32{ 14, // 74: ateapi.UpdateActorRequest.actor:type_name -> ateapi.Actor 26, // 75: ateapi.SuspendActorRequest.actor:type_name -> ateapi.ObjectRef 14, // 76: ateapi.SuspendActorResponse.actor:type_name -> ateapi.Actor - 26, // 77: ateapi.PauseActorRequest.actor:type_name -> ateapi.ObjectRef - 14, // 78: ateapi.PauseActorResponse.actor:type_name -> ateapi.Actor - 26, // 79: ateapi.ResumeActorRequest.actor:type_name -> ateapi.ObjectRef - 14, // 80: ateapi.ResumeActorResponse.actor:type_name -> ateapi.Actor - 26, // 81: ateapi.DeleteActorRequest.actor:type_name -> ateapi.ObjectRef - 26, // 82: ateapi.GetActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 26, // 83: ateapi.CreateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 15, // 84: ateapi.CreateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy - 26, // 85: ateapi.UpdateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 15, // 86: ateapi.UpdateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy - 26, // 87: ateapi.DeleteActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef - 26, // 88: ateapi.GetTagRequest.tag:type_name -> ateapi.ObjectRef - 24, // 89: ateapi.ListTagsResponse.tags:type_name -> ateapi.Tag - 24, // 90: ateapi.CreateTagRequest.tag:type_name -> ateapi.Tag - 24, // 91: ateapi.UpdateTagRequest.tag:type_name -> ateapi.Tag - 26, // 92: ateapi.DeleteTagRequest.tag:type_name -> ateapi.ObjectRef - 26, // 93: ateapi.ListWorkerActorAssignmentsRequest.worker:type_name -> ateapi.ObjectRef - 96, // 94: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment - 93, // 95: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker - 26, // 96: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef - 93, // 97: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker - 93, // 98: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker - 26, // 99: ateapi.DeleteWorkerRequest.worker:type_name -> ateapi.ObjectRef - 81, // 100: ateapi.DeleteWorkerRequest.options:type_name -> ateapi.DeleteOptions - 26, // 101: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef - 14, // 102: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor - 12, // 103: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata - 105, // 104: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry - 94, // 105: ateapi.Worker.status:type_name -> ateapi.WorkerStatus - 6, // 106: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState - 95, // 107: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources - 95, // 108: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources - 28, // 109: ateapi.WorkerResources.resources:type_name -> ateapi.Resources - 12, // 110: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata - 26, // 111: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef - 26, // 112: ateapi.ActorAssignment.actor_template_ref:type_name -> ateapi.ObjectRef - 28, // 113: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources - 26, // 114: ateapi.SetWorkerCapacityRequest.worker:type_name -> ateapi.ObjectRef - 95, // 115: ateapi.SetWorkerCapacityRequest.capacity:type_name -> ateapi.WorkerResources - 93, // 116: ateapi.SetWorkerCapacityResponse.worker:type_name -> ateapi.Worker - 26, // 117: ateapi.MintCertRequest.worker:type_name -> ateapi.ObjectRef - 7, // 118: ateapi.MintCertRequest.purpose:type_name -> ateapi.ActorCertificatePurpose - 61, // 119: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest - 62, // 120: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest - 63, // 121: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest - 64, // 122: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest - 66, // 123: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest - 68, // 124: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest - 70, // 125: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest - 71, // 126: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest - 72, // 127: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest - 73, // 128: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest - 74, // 129: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest - 78, // 130: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest - 75, // 131: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest - 76, // 132: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest - 79, // 133: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest - 80, // 134: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest - 84, // 135: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest - 86, // 136: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest - 87, // 137: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest - 88, // 138: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest - 89, // 139: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest - 90, // 140: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest - 82, // 141: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest - 91, // 142: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest - 51, // 143: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest - 52, // 144: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest - 53, // 145: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest - 55, // 146: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest - 56, // 147: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest - 57, // 148: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest - 58, // 149: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest - 60, // 150: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest - 99, // 151: ateapi.ActorIdentity.MintJWT:input_type -> ateapi.MintJWTRequest - 101, // 152: ateapi.ActorIdentity.MintCert:input_type -> ateapi.MintCertRequest - 97, // 153: ateapi.WorkerService.SetWorkerCapacity:input_type -> ateapi.SetWorkerCapacityRequest - 14, // 154: ateapi.Control.GetActor:output_type -> ateapi.Actor - 14, // 155: ateapi.Control.CreateActor:output_type -> ateapi.Actor - 14, // 156: ateapi.Control.UpdateActor:output_type -> ateapi.Actor - 65, // 157: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse - 67, // 158: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse - 69, // 159: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse - 14, // 160: ateapi.Control.DeleteActor:output_type -> ateapi.Actor - 15, // 161: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy - 15, // 162: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 15, // 163: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy - 15, // 164: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy - 24, // 165: ateapi.Control.CreateTag:output_type -> ateapi.Tag - 24, // 166: ateapi.Control.GetTag:output_type -> ateapi.Tag - 77, // 167: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse - 24, // 168: ateapi.Control.UpdateTag:output_type -> ateapi.Tag - 24, // 169: ateapi.Control.DeleteTag:output_type -> ateapi.Tag - 85, // 170: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse - 93, // 171: ateapi.Control.GetWorker:output_type -> ateapi.Worker - 93, // 172: ateapi.Control.CreateWorker:output_type -> ateapi.Worker - 93, // 173: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker - 93, // 174: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker - 93, // 175: ateapi.Control.DrainWorker:output_type -> ateapi.Worker - 83, // 176: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse - 92, // 177: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse - 25, // 178: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace - 25, // 179: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace - 54, // 180: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse - 25, // 181: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace - 27, // 182: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate - 27, // 183: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate - 59, // 184: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse - 27, // 185: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate - 100, // 186: ateapi.ActorIdentity.MintJWT:output_type -> ateapi.MintJWTResponse - 102, // 187: ateapi.ActorIdentity.MintCert:output_type -> ateapi.MintCertResponse - 98, // 188: ateapi.WorkerService.SetWorkerCapacity:output_type -> ateapi.SetWorkerCapacityResponse - 154, // [154:189] is the sub-list for method output_type - 119, // [119:154] is the sub-list for method input_type - 119, // [119:119] is the sub-list for extension type_name - 119, // [119:119] is the sub-list for extension extendee - 0, // [0:119] is the sub-list for field type_name + 26, // 77: ateapi.SuspendActorWithLeaseRequest.actor:type_name -> ateapi.ObjectRef + 71, // 78: ateapi.SuspendActorWithLeaseRequest.lease:type_name -> ateapi.ActorLease + 26, // 79: ateapi.PauseActorRequest.actor:type_name -> ateapi.ObjectRef + 14, // 80: ateapi.PauseActorResponse.actor:type_name -> ateapi.Actor + 26, // 81: ateapi.ResumeActorRequest.actor:type_name -> ateapi.ObjectRef + 71, // 82: ateapi.ResumeActorRequest.lease:type_name -> ateapi.ActorLease + 14, // 83: ateapi.ResumeActorResponse.actor:type_name -> ateapi.Actor + 71, // 84: ateapi.ResumeActorResponse.lease:type_name -> ateapi.ActorLease + 110, // 85: ateapi.ActorLease.expires_at:type_name -> google.protobuf.Timestamp + 26, // 86: ateapi.RenewActorLeaseRequest.actor:type_name -> ateapi.ObjectRef + 71, // 87: ateapi.RenewActorLeaseRequest.lease:type_name -> ateapi.ActorLease + 71, // 88: ateapi.RenewActorLeaseResponse.lease:type_name -> ateapi.ActorLease + 26, // 89: ateapi.DeleteActorRequest.actor:type_name -> ateapi.ObjectRef + 26, // 90: ateapi.GetActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 26, // 91: ateapi.CreateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 15, // 92: ateapi.CreateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy + 26, // 93: ateapi.UpdateActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 15, // 94: ateapi.UpdateActorEgressPolicyRequest.egress_policy:type_name -> ateapi.EgressPolicy + 26, // 95: ateapi.DeleteActorEgressPolicyRequest.actor:type_name -> ateapi.ObjectRef + 26, // 96: ateapi.GetTagRequest.tag:type_name -> ateapi.ObjectRef + 24, // 97: ateapi.ListTagsResponse.tags:type_name -> ateapi.Tag + 24, // 98: ateapi.CreateTagRequest.tag:type_name -> ateapi.Tag + 24, // 99: ateapi.UpdateTagRequest.tag:type_name -> ateapi.Tag + 26, // 100: ateapi.DeleteTagRequest.tag:type_name -> ateapi.ObjectRef + 26, // 101: ateapi.ListWorkerActorAssignmentsRequest.worker:type_name -> ateapi.ObjectRef + 100, // 102: ateapi.ListWorkerActorAssignmentsResponse.actor_assignments:type_name -> ateapi.ActorAssignment + 97, // 103: ateapi.ListWorkersResponse.workers:type_name -> ateapi.Worker + 26, // 104: ateapi.GetWorkerRequest.worker:type_name -> ateapi.ObjectRef + 97, // 105: ateapi.CreateWorkerRequest.worker:type_name -> ateapi.Worker + 97, // 106: ateapi.UpdateWorkerRequest.worker:type_name -> ateapi.Worker + 26, // 107: ateapi.DeleteWorkerRequest.worker:type_name -> ateapi.ObjectRef + 85, // 108: ateapi.DeleteWorkerRequest.options:type_name -> ateapi.DeleteOptions + 26, // 109: ateapi.DrainWorkerRequest.worker:type_name -> ateapi.ObjectRef + 14, // 110: ateapi.ListActorsResponse.actors:type_name -> ateapi.Actor + 12, // 111: ateapi.Worker.metadata:type_name -> ateapi.ResourceMetadata + 109, // 112: ateapi.Worker.labels:type_name -> ateapi.Worker.LabelsEntry + 98, // 113: ateapi.Worker.status:type_name -> ateapi.WorkerStatus + 6, // 114: ateapi.WorkerStatus.state:type_name -> ateapi.WorkerState + 99, // 115: ateapi.WorkerStatus.capacity:type_name -> ateapi.WorkerResources + 99, // 116: ateapi.WorkerStatus.allocated:type_name -> ateapi.WorkerResources + 28, // 117: ateapi.WorkerResources.resources:type_name -> ateapi.Resources + 12, // 118: ateapi.ActorAssignment.metadata:type_name -> ateapi.ResourceMetadata + 26, // 119: ateapi.ActorAssignment.actor:type_name -> ateapi.ObjectRef + 26, // 120: ateapi.ActorAssignment.actor_template_ref:type_name -> ateapi.ObjectRef + 28, // 121: ateapi.ActorAssignment.resources:type_name -> ateapi.Resources + 26, // 122: ateapi.SetWorkerCapacityRequest.worker:type_name -> ateapi.ObjectRef + 99, // 123: ateapi.SetWorkerCapacityRequest.capacity:type_name -> ateapi.WorkerResources + 97, // 124: ateapi.SetWorkerCapacityResponse.worker:type_name -> ateapi.Worker + 26, // 125: ateapi.MintCertRequest.worker:type_name -> ateapi.ObjectRef + 7, // 126: ateapi.MintCertRequest.purpose:type_name -> ateapi.ActorCertificatePurpose + 61, // 127: ateapi.Control.GetActor:input_type -> ateapi.GetActorRequest + 62, // 128: ateapi.Control.CreateActor:input_type -> ateapi.CreateActorRequest + 63, // 129: ateapi.Control.UpdateActor:input_type -> ateapi.UpdateActorRequest + 64, // 130: ateapi.Control.SuspendActor:input_type -> ateapi.SuspendActorRequest + 66, // 131: ateapi.Control.SuspendActorWithLease:input_type -> ateapi.SuspendActorWithLeaseRequest + 67, // 132: ateapi.Control.PauseActor:input_type -> ateapi.PauseActorRequest + 69, // 133: ateapi.Control.ResumeActor:input_type -> ateapi.ResumeActorRequest + 72, // 134: ateapi.Control.RenewActorLease:input_type -> ateapi.RenewActorLeaseRequest + 74, // 135: ateapi.Control.DeleteActor:input_type -> ateapi.DeleteActorRequest + 75, // 136: ateapi.Control.GetActorEgressPolicy:input_type -> ateapi.GetActorEgressPolicyRequest + 76, // 137: ateapi.Control.CreateActorEgressPolicy:input_type -> ateapi.CreateActorEgressPolicyRequest + 77, // 138: ateapi.Control.UpdateActorEgressPolicy:input_type -> ateapi.UpdateActorEgressPolicyRequest + 78, // 139: ateapi.Control.DeleteActorEgressPolicy:input_type -> ateapi.DeleteActorEgressPolicyRequest + 82, // 140: ateapi.Control.CreateTag:input_type -> ateapi.CreateTagRequest + 79, // 141: ateapi.Control.GetTag:input_type -> ateapi.GetTagRequest + 80, // 142: ateapi.Control.ListTags:input_type -> ateapi.ListTagsRequest + 83, // 143: ateapi.Control.UpdateTag:input_type -> ateapi.UpdateTagRequest + 84, // 144: ateapi.Control.DeleteTag:input_type -> ateapi.DeleteTagRequest + 88, // 145: ateapi.Control.ListWorkers:input_type -> ateapi.ListWorkersRequest + 90, // 146: ateapi.Control.GetWorker:input_type -> ateapi.GetWorkerRequest + 91, // 147: ateapi.Control.CreateWorker:input_type -> ateapi.CreateWorkerRequest + 92, // 148: ateapi.Control.UpdateWorker:input_type -> ateapi.UpdateWorkerRequest + 93, // 149: ateapi.Control.DeleteWorker:input_type -> ateapi.DeleteWorkerRequest + 94, // 150: ateapi.Control.DrainWorker:input_type -> ateapi.DrainWorkerRequest + 86, // 151: ateapi.Control.ListWorkerActorAssignments:input_type -> ateapi.ListWorkerActorAssignmentsRequest + 95, // 152: ateapi.Control.ListActors:input_type -> ateapi.ListActorsRequest + 51, // 153: ateapi.Control.CreateAtespace:input_type -> ateapi.CreateAtespaceRequest + 52, // 154: ateapi.Control.GetAtespace:input_type -> ateapi.GetAtespaceRequest + 53, // 155: ateapi.Control.ListAtespaces:input_type -> ateapi.ListAtespacesRequest + 55, // 156: ateapi.Control.DeleteAtespace:input_type -> ateapi.DeleteAtespaceRequest + 56, // 157: ateapi.Control.CreateActorTemplate:input_type -> ateapi.CreateActorTemplateRequest + 57, // 158: ateapi.Control.GetActorTemplate:input_type -> ateapi.GetActorTemplateRequest + 58, // 159: ateapi.Control.ListActorTemplates:input_type -> ateapi.ListActorTemplatesRequest + 60, // 160: ateapi.Control.DeleteActorTemplate:input_type -> ateapi.DeleteActorTemplateRequest + 103, // 161: ateapi.ActorIdentity.MintJWT:input_type -> ateapi.MintJWTRequest + 105, // 162: ateapi.ActorIdentity.MintCert:input_type -> ateapi.MintCertRequest + 101, // 163: ateapi.WorkerService.SetWorkerCapacity:input_type -> ateapi.SetWorkerCapacityRequest + 14, // 164: ateapi.Control.GetActor:output_type -> ateapi.Actor + 14, // 165: ateapi.Control.CreateActor:output_type -> ateapi.Actor + 14, // 166: ateapi.Control.UpdateActor:output_type -> ateapi.Actor + 65, // 167: ateapi.Control.SuspendActor:output_type -> ateapi.SuspendActorResponse + 65, // 168: ateapi.Control.SuspendActorWithLease:output_type -> ateapi.SuspendActorResponse + 68, // 169: ateapi.Control.PauseActor:output_type -> ateapi.PauseActorResponse + 70, // 170: ateapi.Control.ResumeActor:output_type -> ateapi.ResumeActorResponse + 73, // 171: ateapi.Control.RenewActorLease:output_type -> ateapi.RenewActorLeaseResponse + 14, // 172: ateapi.Control.DeleteActor:output_type -> ateapi.Actor + 15, // 173: ateapi.Control.GetActorEgressPolicy:output_type -> ateapi.EgressPolicy + 15, // 174: ateapi.Control.CreateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 15, // 175: ateapi.Control.UpdateActorEgressPolicy:output_type -> ateapi.EgressPolicy + 15, // 176: ateapi.Control.DeleteActorEgressPolicy:output_type -> ateapi.EgressPolicy + 24, // 177: ateapi.Control.CreateTag:output_type -> ateapi.Tag + 24, // 178: ateapi.Control.GetTag:output_type -> ateapi.Tag + 81, // 179: ateapi.Control.ListTags:output_type -> ateapi.ListTagsResponse + 24, // 180: ateapi.Control.UpdateTag:output_type -> ateapi.Tag + 24, // 181: ateapi.Control.DeleteTag:output_type -> ateapi.Tag + 89, // 182: ateapi.Control.ListWorkers:output_type -> ateapi.ListWorkersResponse + 97, // 183: ateapi.Control.GetWorker:output_type -> ateapi.Worker + 97, // 184: ateapi.Control.CreateWorker:output_type -> ateapi.Worker + 97, // 185: ateapi.Control.UpdateWorker:output_type -> ateapi.Worker + 97, // 186: ateapi.Control.DeleteWorker:output_type -> ateapi.Worker + 97, // 187: ateapi.Control.DrainWorker:output_type -> ateapi.Worker + 87, // 188: ateapi.Control.ListWorkerActorAssignments:output_type -> ateapi.ListWorkerActorAssignmentsResponse + 96, // 189: ateapi.Control.ListActors:output_type -> ateapi.ListActorsResponse + 25, // 190: ateapi.Control.CreateAtespace:output_type -> ateapi.Atespace + 25, // 191: ateapi.Control.GetAtespace:output_type -> ateapi.Atespace + 54, // 192: ateapi.Control.ListAtespaces:output_type -> ateapi.ListAtespacesResponse + 25, // 193: ateapi.Control.DeleteAtespace:output_type -> ateapi.Atespace + 27, // 194: ateapi.Control.CreateActorTemplate:output_type -> ateapi.ActorTemplate + 27, // 195: ateapi.Control.GetActorTemplate:output_type -> ateapi.ActorTemplate + 59, // 196: ateapi.Control.ListActorTemplates:output_type -> ateapi.ListActorTemplatesResponse + 27, // 197: ateapi.Control.DeleteActorTemplate:output_type -> ateapi.ActorTemplate + 104, // 198: ateapi.ActorIdentity.MintJWT:output_type -> ateapi.MintJWTResponse + 106, // 199: ateapi.ActorIdentity.MintCert:output_type -> ateapi.MintCertResponse + 102, // 200: ateapi.WorkerService.SetWorkerCapacity:output_type -> ateapi.SetWorkerCapacityResponse + 164, // [164:201] is the sub-list for method output_type + 127, // [127:164] is the sub-list for method input_type + 127, // [127:127] is the sub-list for extension type_name + 127, // [127:127] is the sub-list for extension extendee + 0, // [0:127] is the sub-list for field type_name } func init() { file_ateapi_proto_init() } @@ -7510,7 +7802,7 @@ func file_ateapi_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateapi_proto_rawDesc), len(file_ateapi_proto_rawDesc)), NumEnums: 9, - NumMessages: 97, + NumMessages: 101, NumExtensions: 0, NumServices: 3, }, diff --git a/pkg/proto/ateapipb/ateapi.proto b/pkg/proto/ateapipb/ateapi.proto index a581f77796..aba273dae6 100644 --- a/pkg/proto/ateapipb/ateapi.proto +++ b/pkg/proto/ateapipb/ateapi.proto @@ -37,12 +37,18 @@ service Control { // to the template's commit scope where required (Full capture, Data commit). rpc SuspendActor(SuspendActorRequest) returns (SuspendActorResponse) {} + // Suspend an actor only when the caller holds its runtime lease. + rpc SuspendActorWithLease(SuspendActorWithLeaseRequest) returns (SuspendActorResponse) {} + // Pause a given actor and keep its snapshots on node VM. rpc PauseActor(PauseActorRequest) returns (PauseActorResponse) {} // Resume an actor from its latest snapshot. rpc ResumeActor(ResumeActorRequest) returns (ResumeActorResponse) {} + // Renew an actor's runtime lease. + rpc RenewActorLease(RenewActorLeaseRequest) returns (RenewActorLeaseResponse) {} + // Delete an actor. Only suspended actors can be deleted. rpc DeleteActor(DeleteActorRequest) returns (Actor) {} @@ -1340,6 +1346,17 @@ message SuspendActorResponse { Actor actor = 1; } +message SuspendActorWithLeaseRequest { + // +k8s:required + // +k8s:subfield(atespace)=+k8s:required + ObjectRef actor = 1; + + // The runtime lease returned by ResumeActor. + // + // +k8s:required + ActorLease lease = 2; +} + message PauseActorRequest { // +k8s:required // +k8s:subfield(atespace)=+k8s:required @@ -1359,6 +1376,18 @@ message ResumeActorRequest { // // +k8s:optional bool boot = 2; + + // Reattach to an already-running actor only with its current runtime lease. + // + // +k8s:optional + ActorLease lease = 3; + + // Claim the actor's runtime lease for an executor-owned workload. The + // legacy router path leaves this false so it can keep resolving already + // running actors without becoming their liveness owner. + // + // +k8s:optional + bool claim_runtime_lease = 4; } message ResumeActorResponse { @@ -1367,6 +1396,36 @@ message ResumeActorResponse { // True if a resume workflow was executed to activate the actor. // False if the actor was already RUNNING. bool resumed = 2; + + // The runtime lease for the running actor. + ActorLease lease = 3; +} + +// ActorLease identifies one actor workload incarnation. The token is opaque to +// callers; generation prevents an old token from being reused after reclaim. +message ActorLease { + // +k8s:required + string token = 1; + + // +k8s:required + // +k8s:minimum=1 + int64 generation = 2; + + // Server-assigned expiration time. + google.protobuf.Timestamp expires_at = 3; +} + +message RenewActorLeaseRequest { + // +k8s:required + // +k8s:subfield(atespace)=+k8s:required + ObjectRef actor = 1; + + // +k8s:required + ActorLease lease = 2; +} + +message RenewActorLeaseResponse { + ActorLease lease = 1; } message DeleteActorRequest { diff --git a/pkg/proto/ateapipb/ateapi_grpc.pb.go b/pkg/proto/ateapipb/ateapi_grpc.pb.go index fcfbe31cc0..d27b4cfa38 100644 --- a/pkg/proto/ateapipb/ateapi_grpc.pb.go +++ b/pkg/proto/ateapipb/ateapi_grpc.pb.go @@ -37,8 +37,10 @@ const ( Control_CreateActor_FullMethodName = "/ateapi.Control/CreateActor" Control_UpdateActor_FullMethodName = "/ateapi.Control/UpdateActor" Control_SuspendActor_FullMethodName = "/ateapi.Control/SuspendActor" + Control_SuspendActorWithLease_FullMethodName = "/ateapi.Control/SuspendActorWithLease" Control_PauseActor_FullMethodName = "/ateapi.Control/PauseActor" Control_ResumeActor_FullMethodName = "/ateapi.Control/ResumeActor" + Control_RenewActorLease_FullMethodName = "/ateapi.Control/RenewActorLease" Control_DeleteActor_FullMethodName = "/ateapi.Control/DeleteActor" Control_GetActorEgressPolicy_FullMethodName = "/ateapi.Control/GetActorEgressPolicy" Control_CreateActorEgressPolicy_FullMethodName = "/ateapi.Control/CreateActorEgressPolicy" @@ -83,10 +85,14 @@ type ControlClient interface { // on its worker; a paused actor's node-local snapshot is uploaded, narrowed // to the template's commit scope where required (Full capture, Data commit). SuspendActor(ctx context.Context, in *SuspendActorRequest, opts ...grpc.CallOption) (*SuspendActorResponse, error) + // Suspend an actor only when the caller holds its runtime lease. + SuspendActorWithLease(ctx context.Context, in *SuspendActorWithLeaseRequest, opts ...grpc.CallOption) (*SuspendActorResponse, error) // Pause a given actor and keep its snapshots on node VM. PauseActor(ctx context.Context, in *PauseActorRequest, opts ...grpc.CallOption) (*PauseActorResponse, error) // Resume an actor from its latest snapshot. ResumeActor(ctx context.Context, in *ResumeActorRequest, opts ...grpc.CallOption) (*ResumeActorResponse, error) + // Renew an actor's runtime lease. + RenewActorLease(ctx context.Context, in *RenewActorLeaseRequest, opts ...grpc.CallOption) (*RenewActorLeaseResponse, error) // Delete an actor. Only suspended actors can be deleted. DeleteActor(ctx context.Context, in *DeleteActorRequest, opts ...grpc.CallOption) (*Actor, error) // Get the egress policy resource nested under an Actor. @@ -195,6 +201,16 @@ func (c *controlClient) SuspendActor(ctx context.Context, in *SuspendActorReques return out, nil } +func (c *controlClient) SuspendActorWithLease(ctx context.Context, in *SuspendActorWithLeaseRequest, opts ...grpc.CallOption) (*SuspendActorResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SuspendActorResponse) + err := c.cc.Invoke(ctx, Control_SuspendActorWithLease_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *controlClient) PauseActor(ctx context.Context, in *PauseActorRequest, opts ...grpc.CallOption) (*PauseActorResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PauseActorResponse) @@ -215,6 +231,16 @@ func (c *controlClient) ResumeActor(ctx context.Context, in *ResumeActorRequest, return out, nil } +func (c *controlClient) RenewActorLease(ctx context.Context, in *RenewActorLeaseRequest, opts ...grpc.CallOption) (*RenewActorLeaseResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RenewActorLeaseResponse) + err := c.cc.Invoke(ctx, Control_RenewActorLease_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *controlClient) DeleteActor(ctx context.Context, in *DeleteActorRequest, opts ...grpc.CallOption) (*Actor, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Actor) @@ -491,10 +517,14 @@ type ControlServer interface { // on its worker; a paused actor's node-local snapshot is uploaded, narrowed // to the template's commit scope where required (Full capture, Data commit). SuspendActor(context.Context, *SuspendActorRequest) (*SuspendActorResponse, error) + // Suspend an actor only when the caller holds its runtime lease. + SuspendActorWithLease(context.Context, *SuspendActorWithLeaseRequest) (*SuspendActorResponse, error) // Pause a given actor and keep its snapshots on node VM. PauseActor(context.Context, *PauseActorRequest) (*PauseActorResponse, error) // Resume an actor from its latest snapshot. ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) + // Renew an actor's runtime lease. + RenewActorLease(context.Context, *RenewActorLeaseRequest) (*RenewActorLeaseResponse, error) // Delete an actor. Only suspended actors can be deleted. DeleteActor(context.Context, *DeleteActorRequest) (*Actor, error) // Get the egress policy resource nested under an Actor. @@ -575,12 +605,18 @@ func (UnimplementedControlServer) UpdateActor(context.Context, *UpdateActorReque func (UnimplementedControlServer) SuspendActor(context.Context, *SuspendActorRequest) (*SuspendActorResponse, error) { return nil, status.Error(codes.Unimplemented, "method SuspendActor not implemented") } +func (UnimplementedControlServer) SuspendActorWithLease(context.Context, *SuspendActorWithLeaseRequest) (*SuspendActorResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SuspendActorWithLease not implemented") +} func (UnimplementedControlServer) PauseActor(context.Context, *PauseActorRequest) (*PauseActorResponse, error) { return nil, status.Error(codes.Unimplemented, "method PauseActor not implemented") } func (UnimplementedControlServer) ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) { return nil, status.Error(codes.Unimplemented, "method ResumeActor not implemented") } +func (UnimplementedControlServer) RenewActorLease(context.Context, *RenewActorLeaseRequest) (*RenewActorLeaseResponse, error) { + return nil, status.Error(codes.Unimplemented, "method RenewActorLease not implemented") +} func (UnimplementedControlServer) DeleteActor(context.Context, *DeleteActorRequest) (*Actor, error) { return nil, status.Error(codes.Unimplemented, "method DeleteActor not implemented") } @@ -752,6 +788,24 @@ func _Control_SuspendActor_Handler(srv interface{}, ctx context.Context, dec fun return interceptor(ctx, in, info, handler) } +func _Control_SuspendActorWithLease_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SuspendActorWithLeaseRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).SuspendActorWithLease(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_SuspendActorWithLease_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).SuspendActorWithLease(ctx, req.(*SuspendActorWithLeaseRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Control_PauseActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(PauseActorRequest) if err := dec(in); err != nil { @@ -788,6 +842,24 @@ func _Control_ResumeActor_Handler(srv interface{}, ctx context.Context, dec func return interceptor(ctx, in, info, handler) } +func _Control_RenewActorLease_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RenewActorLeaseRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).RenewActorLease(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_RenewActorLease_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).RenewActorLease(ctx, req.(*RenewActorLeaseRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Control_DeleteActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DeleteActorRequest) if err := dec(in); err != nil { @@ -1279,6 +1351,10 @@ var Control_ServiceDesc = grpc.ServiceDesc{ MethodName: "SuspendActor", Handler: _Control_SuspendActor_Handler, }, + { + MethodName: "SuspendActorWithLease", + Handler: _Control_SuspendActorWithLease_Handler, + }, { MethodName: "PauseActor", Handler: _Control_PauseActor_Handler, @@ -1287,6 +1363,10 @@ var Control_ServiceDesc = grpc.ServiceDesc{ MethodName: "ResumeActor", Handler: _Control_ResumeActor_Handler, }, + { + MethodName: "RenewActorLease", + Handler: _Control_RenewActorLease_Handler, + }, { MethodName: "DeleteActor", Handler: _Control_DeleteActor_Handler, diff --git a/tools/setup-gcp/README.md b/tools/setup-gcp/README.md index fe761fe966..4dc9bc3df9 100644 --- a/tools/setup-gcp/README.md +++ b/tools/setup-gcp/README.md @@ -95,6 +95,34 @@ Filestore CSI driver disabled). > podcertificate ClusterTrustBundles to be ready" and `kubectl get > clustertrustbundles` reports the resource type is not served. +> [!WARNING] +> **Turn node auto-upgrade off on any node pool that runs workers, and do not +> use spot or preemptible nodes for them.** When a worker pod is deleted, +> `SIGTERM` is forwarded into the actor's containers and the control plane keeps +> accepting a suspend for about 60 seconds. An actor suspended inside that +> window keeps its state. One still awake when the window closes is moved to +> `ACTOR_STATE_CRASHED` with its worker assignment cleared, and `CRASHED` is +> terminal: `resume` and `suspend` are both refused, there is no recover verb, +> and the snapshot the actor still holds cannot be used to start it. The only +> way out is to delete the actor and create a new one, which loses its state. +> +> Auto-upgrade is the trigger to plan for, because GKE enables it by default and +> it fires on Google's maintenance schedule rather than yours. `create cluster` +> does not disable it, so do it yourself on every pool that runs workers: +> +> ```bash +> gcloud container node-pools update "${NODE_POOL}" \ +> --cluster "${CLUSTER_NAME}" --location "${CLUSTER_LOCATION}" \ +> --no-enable-autoupgrade +> ``` +> +> This is a management setting, so it takes effect without recreating nodes and +> is safe to apply to a serving cluster. Node auto-repair, preemption and OOM +> kills reach the same path and cannot be configured away, so treat the setting +> as removing the scheduled risk rather than all of it. Change versions through +> the [rolling upgrade runbook](../../docs/upgrade.md), which has you suspend +> every actor on a node at your own pace before the node moves. + ```bash go run ./tools/setup-gcp create cluster [flags] ```