diff --git a/CHANGELOG.md b/CHANGELOG.md index e1191e015..45564c437 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.15.1-patch-v2] + +### Added + +- Release the existing paused-only `PUT /vm/pre-fault-memory` API from PR #18. + On x86_64 with `KVM_CAP_PRE_FAULT_MEMORY`, it distributes validated, + page-aligned guest-physical ranges across the available vCPUs using + `KVM_PRE_FAULT_MEMORY`; aarch64 remains unsupported. This release does not + change startup, restore behavior, or the snapshot serialization/data format. + ## [1.15.1] ### Added diff --git a/Cargo.lock b/Cargo.lock index 3750df127..2b4b420d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -383,7 +383,7 @@ dependencies = [ [[package]] name = "cpu-template-helper" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "clap", "displaydoc", @@ -550,7 +550,7 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "firecracker" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "cargo_toml", "displaydoc", @@ -704,7 +704,7 @@ checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jailer" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "libc", "log-instrument", @@ -1073,7 +1073,7 @@ dependencies = [ [[package]] name = "rebase-snap" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "displaydoc", "libc", @@ -1157,7 +1157,7 @@ dependencies = [ [[package]] name = "seccompiler" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "bitcode", "clap", @@ -1254,7 +1254,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "snapshot-editor" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" dependencies = [ "clap", "clap-num", diff --git a/src/cpu-template-helper/Cargo.toml b/src/cpu-template-helper/Cargo.toml index a856bf216..689a5388e 100644 --- a/src/cpu-template-helper/Cargo.toml +++ b/src/cpu-template-helper/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cpu-template-helper" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" license = "Apache-2.0" diff --git a/src/firecracker/Cargo.toml b/src/firecracker/Cargo.toml index 1e3a7b483..20f999738 100644 --- a/src/firecracker/Cargo.toml +++ b/src/firecracker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "firecracker" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" build = "build.rs" diff --git a/src/firecracker/src/api_server/parsed_request.rs b/src/firecracker/src/api_server/parsed_request.rs index 50a38cebd..405fecfc4 100644 --- a/src/firecracker/src/api_server/parsed_request.rs +++ b/src/firecracker/src/api_server/parsed_request.rs @@ -89,6 +89,9 @@ impl TryFrom<&Request> for ParsedRequest { Some("dirty-memory-ranges") => { Ok(ParsedRequest::new_sync(VmmAction::GetDirtyMemoryRanges)) } + Some("resident-memory-ranges") => { + Ok(ParsedRequest::new_sync(VmmAction::GetResidentMemoryRanges)) + } Some("guest-memory-regions") => { Ok(ParsedRequest::new_sync(VmmAction::GetGuestMemoryRegions)) } @@ -214,7 +217,9 @@ impl ParsedRequest { ), VmmData::FullVmConfig(config) => Self::success_response_with_data(config), VmmData::DirtyMemoryRanges(ranges) => Self::success_response_with_data(ranges), + VmmData::ResidentMemoryRanges(ranges) => Self::success_response_with_data(ranges), VmmData::GuestMemoryRegions(regions) => Self::success_response_with_data(regions), + VmmData::PreFaultMemoryStats(stats) => Self::success_response_with_data(stats), }, Err(vmm_action_error) => { let mut response = match vmm_action_error { @@ -367,7 +372,9 @@ pub mod tests { use vmm::vmm_config::balloon::{BalloonDeviceConfig, BalloonStats}; use vmm::vmm_config::instance_info::InstanceInfo; use vmm::vmm_config::machine_config::MachineConfig; - use vmm::vstate::memory::{DirtyMemoryRange, DirtyMemoryRanges}; + use vmm::vstate::memory::{ + DirtyMemoryRange, DirtyMemoryRanges, ResidentMemoryRange, ResidentMemoryRanges, + }; use super::*; @@ -690,9 +697,15 @@ pub mod tests { VmmData::DirtyMemoryRanges(ranges) => { http_response(&serde_json::to_string(ranges).unwrap(), 200) } + VmmData::ResidentMemoryRanges(ranges) => { + http_response(&serde_json::to_string(ranges).unwrap(), 200) + } VmmData::GuestMemoryRegions(regions) => { http_response(&serde_json::to_string(regions).unwrap(), 200) } + VmmData::PreFaultMemoryStats(stats) => { + http_response(&serde_json::to_string(stats).unwrap(), 200) + } VmmData::MachineConfiguration(cfg) => { http_response(&serde_json::to_string(cfg).unwrap(), 200) } @@ -732,6 +745,15 @@ pub mod tests { length: 4096, }], })); + verify_ok_response_with(VmmData::ResidentMemoryRanges(ResidentMemoryRanges { + page_size: 4096, + memory_size: 8192, + ranges: vec![ResidentMemoryRange { + base_host_virt_addr: 0x7f0000000000, + image_offset: 0, + length: 4096, + }], + })); verify_ok_response_with(VmmData::MachineConfiguration(MachineConfig::default())); verify_ok_response_with(VmmData::MmdsValue(serde_json::from_str("{}").unwrap())); verify_ok_response_with(VmmData::InstanceInformation(InstanceInfo::default())); @@ -739,6 +761,7 @@ pub mod tests { #[allow(deprecated)] verify_ok_response_with(VmmData::GuestMemoryRegions(vec![GuestRegionUffdMapping { base_host_virt_addr: 0x7f0000000000, + guest_phys_addr: 0, size: 0x10000000, offset: 0, page_size: 4096, @@ -872,6 +895,22 @@ pub mod tests { ); } + #[test] + fn test_try_from_get_resident_memory_ranges() { + let (mut sender, receiver) = UnixStream::pair().unwrap(); + let mut connection = HttpConnection::new(receiver); + sender + .write_all(http_request("GET", "/vm/resident-memory-ranges", None).as_bytes()) + .unwrap(); + connection.try_read().unwrap(); + let req = connection.pop_parsed_request().unwrap(); + let parsed = ParsedRequest::try_from(&req).unwrap(); + assert_eq!( + vmm_action_from_request(parsed), + VmmAction::GetResidentMemoryRanges + ); + } + #[test] fn test_try_from_put_actions() { let (mut sender, receiver) = UnixStream::pair().unwrap(); diff --git a/src/firecracker/swagger/firecracker.yaml b/src/firecracker/swagger/firecracker.yaml index 97ea1ddab..e3507048c 100644 --- a/src/firecracker/swagger/firecracker.yaml +++ b/src/firecracker/swagger/firecracker.yaml @@ -5,7 +5,7 @@ info: The API is accessible through HTTP calls on specific URLs carrying JSON modeled data. The transport medium is a Unix Domain Socket. - version: 1.15.1-patch-v1 + version: 1.15.1-patch-v2 termsOfService: "" contact: email: "firecracker-maintainers@amazon.com" @@ -843,8 +843,10 @@ paths: schema: $ref: "#/definitions/PreFaultMemoryRequest" responses: - 204: + 200: description: Pre-fault completed successfully. + schema: + $ref: "#/definitions/PreFaultMemoryStats" 400: description: Invalid request, unsupported host capability, or VM state. schema: @@ -906,6 +908,29 @@ paths: schema: $ref: "#/definitions/Error" + /vm/resident-memory-ranges: + get: + summary: Returns resident guest memory ranges. + description: + Returns host-resident guest memory ranges in the same contiguous image + layout used by Firecracker memory snapshots. Residency is sampled with + mincore(2); this API neither reads nor changes KVM dirty-page state. + Post-boot only. + operationId: getResidentMemoryRanges + responses: + 200: + description: Resident guest memory ranges + schema: + $ref: "#/definitions/ResidentMemoryRanges" + 400: + description: VM is not in a valid state for this operation + schema: + $ref: "#/definitions/Error" + default: + description: Internal server error + schema: + $ref: "#/definitions/Error" + /vm/config: get: summary: Gets the full VM configuration. @@ -1357,11 +1382,12 @@ definitions: type: object description: Describes a guest memory region mapping, providing the host virtual address, - region size, offset within a contiguous snapshot layout, and page size. Used - by external processes to read guest memory directly from the Firecracker - process address space via process_vm_readv(). + guest physical address, region size, offset within a contiguous snapshot + layout, and page size. Used by external processes to read guest memory + directly from the Firecracker process address space via process_vm_readv(). required: - base_host_virt_addr + - guest_phys_addr - size - offset - page_size @@ -1370,6 +1396,13 @@ definitions: type: integer format: int64 description: Base host virtual address of the guest memory region. + guest_phys_addr: + type: integer + format: int64 + description: + Guest physical address at which this region begins. Unlike offset, + this preserves holes in the guest physical address space and is used + to translate a host virtual address in the region back to GPAs. size: type: integer format: int64 @@ -1423,6 +1456,77 @@ definitions: items: $ref: "#/definitions/PreFaultMemoryRange" + PreFaultMemoryWorkerStats: + type: object + description: Completion statistics for work assigned to one vCPU worker. + additionalProperties: false + required: + - vcpu_id + - range_count + - requested_bytes + - completed_bytes + - remaining_bytes + - ioctl_count + - wall_time_us + properties: + vcpu_id: + type: integer + format: int32 + range_count: + type: integer + format: int64 + requested_bytes: + type: integer + format: int64 + completed_bytes: + type: integer + format: int64 + remaining_bytes: + type: integer + format: int64 + ioctl_count: + type: integer + format: int64 + wall_time_us: + type: integer + format: int64 + + PreFaultMemoryStats: + type: object + description: Completion statistics for a paused guest-memory pre-fault request. + additionalProperties: false + required: + - range_count + - requested_bytes + - completed_bytes + - remaining_bytes + - ioctl_count + - wall_time_us + - workers + properties: + range_count: + type: integer + format: int64 + requested_bytes: + type: integer + format: int64 + completed_bytes: + type: integer + format: int64 + remaining_bytes: + type: integer + format: int64 + ioctl_count: + type: integer + format: int64 + wall_time_us: + type: integer + format: int64 + workers: + type: array + items: + $ref: "#/definitions/PreFaultMemoryWorkerStats" + DirtyMemoryRange: type: object description: @@ -1470,6 +1574,53 @@ definitions: items: $ref: "#/definitions/DirtyMemoryRange" + ResidentMemoryRange: + type: object + description: + Describes a contiguous host-resident guest-memory range in the + contiguous snapshot image layout. + required: + - base_host_virt_addr + - image_offset + - length + properties: + base_host_virt_addr: + type: integer + format: int64 + description: Base host virtual address of the resident range. + image_offset: + type: integer + format: int64 + description: + Cumulative byte offset of this range in the contiguous memory snapshot + image layout. This is not a Guest Physical Address. + length: + type: integer + format: int64 + description: Resident range length in bytes. + + ResidentMemoryRanges: + type: object + description: + Resident memory ranges for a VM, sampled with mincore(2), using the same + contiguous memory image layout as snapshot files. + required: + - page_size + - memory_size + - ranges + properties: + page_size: + type: integer + description: Page size used by mincore(2). + memory_size: + type: integer + format: int64 + description: Total byte size of the contiguous memory snapshot image. + ranges: + type: array + items: + $ref: "#/definitions/ResidentMemoryRange" + FullVmConfiguration: type: object properties: diff --git a/src/jailer/Cargo.toml b/src/jailer/Cargo.toml index 012c98af5..98c3513b4 100644 --- a/src/jailer/Cargo.toml +++ b/src/jailer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "jailer" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" description = "Process for starting Firecracker in production scenarios; applies a cgroup/namespace isolation barrier and then drops privileges." diff --git a/src/rebase-snap/Cargo.toml b/src/rebase-snap/Cargo.toml index d6abd826b..c292623cb 100644 --- a/src/rebase-snap/Cargo.toml +++ b/src/rebase-snap/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rebase-snap" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" license = "Apache-2.0" diff --git a/src/seccompiler/Cargo.toml b/src/seccompiler/Cargo.toml index a0c66ab13..83dcd27d0 100644 --- a/src/seccompiler/Cargo.toml +++ b/src/seccompiler/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "seccompiler" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" description = "Program that compiles multi-threaded seccomp-bpf filters expressed as JSON into raw BPF programs, serializing them and outputting them to a file." diff --git a/src/snapshot-editor/Cargo.toml b/src/snapshot-editor/Cargo.toml index 8f08a9ac9..46949f60d 100644 --- a/src/snapshot-editor/Cargo.toml +++ b/src/snapshot-editor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "snapshot-editor" -version = "1.15.1-patch-v1" +version = "1.15.1-patch-v2" authors = ["Amazon Firecracker team "] edition = "2024" license = "Apache-2.0" diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index a37a167b3..6a24cc543 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -168,7 +168,9 @@ use crate::vstate::memory::{ GuestAddress, GuestMemory, GuestMemoryExtension, GuestMemoryMmap, GuestMemoryRegion, GuestRegionType, }; -use crate::vstate::prefault::{PreFaultMemoryError, PreFaultMemoryRequest}; +use crate::vstate::prefault::{ + PreFaultMemoryError, PreFaultMemoryRequest, PreFaultMemoryStats, +}; use crate::vstate::prefault::{ drain_pre_fault_responses, send_pre_fault_events, split_pre_fault_ranges, }; @@ -559,7 +561,7 @@ impl Vmm { pub fn pre_fault_memory( &mut self, request: PreFaultMemoryRequest, - ) -> Result<(), PreFaultMemoryError> { + ) -> Result { request.validate()?; if self.instance_info.state != VmState::Paused { @@ -603,27 +605,31 @@ impl Vmm { } let work = split_pre_fault_ranges(&request.ranges, self.vcpus_handles.len())?; + let started = std::time::Instant::now(); // Send every work queue before waiting for any response. This lets the kernel fault pages // on all vCPU threads concurrently. - let send_error = send_pre_fault_events(&work, |vcpu_id, ranges| { - self.vcpus_handles[vcpu_id].send_event(VcpuEvent::PreFaultMemory(ranges)) + let (send_error, successful_vcpu_ids) = send_pre_fault_events(&work, |vcpu_id, ranges| { + self.vcpus_handles[vcpu_id].send_event(VcpuEvent::PreFaultMemory { ranges }) }); - // Drain every response for every worker whose send_event call returned, even after one - // worker fails. Completed kernel work is intentionally not rolled back. This is a - // blocking receive because a fixed timeout could leave a completed response queued and - // have the next Resume/Save operation consume it. - let response_error = drain_pre_fault_responses(work.len(), |vcpu_id| { + // Only drain workers whose event signal succeeded. `send_event` queues before signaling, + // but a signal failure can mean the vCPU thread has already exited and will never send a + // response. Waiting for that worker would leave the paused VM permanently blocked. + let worker_stats = drain_pre_fault_responses(&successful_vcpu_ids, |vcpu_id| { self.vcpus_handles[vcpu_id].response_receiver().recv() }); - match send_error - .map(|(vcpu_id, source)| PreFaultMemoryError::VcpuSend { vcpu_id, source }) - .or(response_error) - { - Some(error) => Err(error), - None => Ok(()), + if let Some((vcpu_id, source)) = send_error { + return Err(PreFaultMemoryError::VcpuSend { vcpu_id, source }); } + let stats = PreFaultMemoryStats::from_workers( + &request.ranges, + worker_stats?, + u64::try_from(started.elapsed().as_micros()).unwrap_or(u64::MAX), + ); + debug_assert_eq!(stats.requested_bytes, stats.completed_bytes); + debug_assert_eq!(stats.remaining_bytes, 0); + Ok(stats) } /// Injects CTRL+ALT+DEL keystroke combo in the i8042 device. diff --git a/src/vmm/src/persist.rs b/src/vmm/src/persist.rs index 8dc8fd85f..5f42b8f65 100644 --- a/src/vmm/src/persist.rs +++ b/src/vmm/src/persist.rs @@ -16,6 +16,7 @@ use std::sync::{Arc, Mutex}; use semver::Version; use serde::{Deserialize, Serialize}; use userfaultfd::{FeatureFlags, Uffd, UffdBuilder}; +use vm_memory::{Address, GuestMemoryRegion}; use vmm_sys_util::sock_ctrl_msg::ScmSocket; #[cfg(target_arch = "aarch64")] @@ -112,6 +113,12 @@ pub struct GuestRegionUffdMapping { /// Base host virtual address where the guest memory contents for this /// region should be copied/populated. pub base_host_virt_addr: u64, + /// Guest physical address at which this memory region begins. + /// + /// Unlike [`Self::offset`], this address preserves holes in the guest + /// physical address space and can therefore be used to translate a host + /// virtual address in this region back to a GPA. + pub guest_phys_addr: u64, /// Region size. pub size: usize, /// Offset in the backend file/buffer where the region contents are. @@ -540,12 +547,12 @@ fn guest_memory_from_uffd( /// Builds a list of [`GuestRegionUffdMapping`]s from an iterator of memory regions. /// -/// Each mapping records the host virtual address, region size, cumulative byte -/// offset within a contiguous layout of all regions, and the page size from the -/// given [`HugePageConfig`]. This is the single source of truth for constructing -/// these mappings — used by both the uffd restore path and the runtime -/// `GET /vm/guest-memory-regions` API. -pub fn build_uffd_mappings<'a, R: Deref + 'a>( +/// Each mapping records the host virtual address, guest physical address, +/// region size, cumulative byte offset within a contiguous layout of all +/// regions, and the page size from the given [`HugePageConfig`]. This is the +/// single source of truth for constructing these mappings — used by both the +/// uffd restore path and the runtime `GET /vm/guest-memory-regions` API. +pub fn build_uffd_mappings<'a, R: Deref + GuestMemoryRegion + 'a>( regions: impl Iterator, huge_pages: HugePageConfig, ) -> Vec { @@ -555,6 +562,7 @@ pub fn build_uffd_mappings<'a, R: Deref + 'a>( #[allow(deprecated)] mappings.push(GuestRegionUffdMapping { base_host_virt_addr: mem_region.as_ptr() as u64, + guest_phys_addr: mem_region.start_addr().raw_value(), size: mem_region.size(), offset, page_size: huge_pages.page_size(), @@ -775,6 +783,7 @@ mod tests { let uffd_regions = vec![ GuestRegionUffdMapping { base_host_virt_addr: 0, + guest_phys_addr: 0, size: 0x100000, offset: 0, page_size: HugePageConfig::None.page_size(), @@ -782,6 +791,7 @@ mod tests { }, GuestRegionUffdMapping { base_host_virt_addr: 0x100000, + guest_phys_addr: 0x200000, size: 0x200000, offset: 0, page_size: HugePageConfig::Hugetlbfs2M.page_size(), @@ -816,7 +826,12 @@ mod tests { use vm_memory::GuestAddress; let regions = memory::anonymous( - [(GuestAddress(0), 0x10000), (GuestAddress(0x10000), 0x20000)] + [ + (GuestAddress(0), 0x10000), + // Model the x86 MMIO hole: snapshot offsets stay contiguous, + // while GPAs must retain the physical-address gap. + (GuestAddress(0x1_0000_0000), 0x20000), + ] .iter() .copied(), false, @@ -830,12 +845,15 @@ mod tests { // First region: offset starts at 0. assert_ne!(mappings[0].base_host_virt_addr, 0); + assert_eq!(mappings[0].guest_phys_addr, 0); assert_eq!(mappings[0].size, 0x10000); assert_eq!(mappings[0].offset, 0); assert_eq!(mappings[0].page_size, 4096); - // Second region: offset is cumulative (== first region's size). + // The second region's image offset is cumulative, but its GPA preserves + // the physical-address gap rather than treating the image as contiguous. assert_ne!(mappings[1].base_host_virt_addr, 0); + assert_eq!(mappings[1].guest_phys_addr, 0x1_0000_0000); assert_eq!(mappings[1].size, 0x20000); assert_eq!(mappings[1].offset, 0x10000); assert_eq!(mappings[1].page_size, 4096); diff --git a/src/vmm/src/rpc_interface.rs b/src/vmm/src/rpc_interface.rs index ffed9c814..b511655eb 100644 --- a/src/vmm/src/rpc_interface.rs +++ b/src/vmm/src/rpc_interface.rs @@ -45,8 +45,10 @@ use crate::vmm_config::serial::SerialConfig; use crate::vmm_config::snapshot::{CreateSnapshotParams, LoadSnapshotParams, SnapshotType}; use crate::vmm_config::vsock::{VsockConfigError, VsockDeviceConfig}; use crate::vmm_config::{self, RateLimiterUpdate}; -use crate::vstate::memory::{DirtyMemoryRanges, GuestMemory}; -use crate::vstate::prefault::{PreFaultMemoryError, PreFaultMemoryRequest}; +use crate::vstate::memory::{DirtyMemoryRanges, GuestMemory, ResidentMemoryRanges}; +use crate::vstate::prefault::{ + PreFaultMemoryError, PreFaultMemoryRequest, PreFaultMemoryStats, +}; use crate::vstate::vm::VmError; /// This enum represents the public interface of the VMM. Each action contains various @@ -75,6 +77,8 @@ pub enum VmmAction { GetFullVmConfig, /// Get dirty guest memory ranges. Post-boot only. GetDirtyMemoryRanges, + /// Get resident guest memory ranges. Post-boot only. + GetResidentMemoryRanges, /// Get guest memory region mappings. Post-boot only. GetGuestMemoryRegions, /// Pre-fault selected guest memory. Paused VM only. @@ -172,6 +176,8 @@ pub enum VmmActionError { CreateSnapshot(#[from] CreateSnapshotError), /// Dirty memory ranges error: {0} DirtyMemoryRanges(#[from] VmError), + /// Resident memory ranges error: {0} + ResidentMemoryRanges(VmError), /// Pre-fault memory error: {0} PreFaultMemory(#[from] PreFaultMemoryError), /// Configure CPU error: {0} @@ -233,8 +239,12 @@ pub enum VmmData { FullVmConfig(VmmConfig), /// The dirty guest memory ranges. DirtyMemoryRanges(DirtyMemoryRanges), + /// The resident guest memory ranges. + ResidentMemoryRanges(ResidentMemoryRanges), /// The guest memory region mappings. GuestMemoryRegions(Vec), + /// Completion statistics for a guest-memory pre-fault request. + PreFaultMemoryStats(PreFaultMemoryStats), /// The microVM configuration represented by `VmConfig`. MachineConfiguration(MachineConfig), /// Mmds contents. @@ -507,6 +517,7 @@ impl<'a> PrebootApiController<'a> { CreateSnapshot(_) | FlushMetrics | GetDirtyMemoryRanges + | GetResidentMemoryRanges | GetGuestMemoryRegions | Pause | Resume @@ -722,6 +733,7 @@ impl RuntimeApiController { self.vmm.lock().expect("Poisoned lock").full_config(), )), GetDirtyMemoryRanges => self.get_dirty_memory_ranges(), + GetResidentMemoryRanges => self.get_resident_memory_ranges(), GetGuestMemoryRegions => self.get_guest_memory_regions(), PreFaultMemory(request) => self.pre_fault_memory(request), GetMemoryHotplugStatus => self @@ -881,7 +893,7 @@ impl RuntimeApiController { .lock() .expect("Poisoned lock") .pre_fault_memory(request) - .map(|()| VmmData::Empty) + .map(VmmData::PreFaultMemoryStats) .map_err(VmmActionError::PreFaultMemory) } @@ -979,6 +991,16 @@ impl RuntimeApiController { Ok(VmmData::DirtyMemoryRanges(ranges)) } + /// Returns host-resident memory ranges for working-set profilers. + fn get_resident_memory_ranges(&self) -> Result { + let locked_vmm = self.vmm.lock().expect("Poisoned lock"); + let ranges = locked_vmm + .vm + .get_resident_memory_ranges() + .map_err(VmmActionError::ResidentMemoryRanges)?; + Ok(VmmData::ResidentMemoryRanges(ranges)) + } + /// Updates block device properties: /// - path of the host file backing the emulated block device, update the disk image on the /// device and its virtio configuration @@ -1215,6 +1237,7 @@ mod tests { check_unsupported(preboot_request(VmmAction::Resume)); check_unsupported(preboot_request(VmmAction::GetBalloonStats)); check_unsupported(preboot_request(VmmAction::GetDirtyMemoryRanges)); + check_unsupported(preboot_request(VmmAction::GetResidentMemoryRanges)); check_unsupported(preboot_request(VmmAction::GetGuestMemoryRegions)); let pre_fault_result = preboot_request(VmmAction::PreFaultMemory( PreFaultMemoryRequest { @@ -1317,6 +1340,18 @@ mod tests { } } + #[test] + fn test_runtime_get_resident_memory_ranges() { + let result = runtime_request(VmmAction::GetResidentMemoryRanges).unwrap(); + match result { + VmmData::ResidentMemoryRanges(resident_ranges) => { + assert_eq!(resident_ranges.page_size, 4096); + assert_ne!(resident_ranges.memory_size, 0); + } + other => panic!("Expected ResidentMemoryRanges, got {:?}", other), + } + } + #[test] fn test_runtime_disallowed() { fn check_unsupported(res: Result) { diff --git a/src/vmm/src/vstate/memory.rs b/src/vmm/src/vstate/memory.rs index 818aa0b8d..ffe0d9966 100644 --- a/src/vmm/src/vstate/memory.rs +++ b/src/vmm/src/vstate/memory.rs @@ -51,6 +51,28 @@ pub struct DirtyMemoryRanges { pub ranges: Vec, } +/// A contiguous resident-memory range in the snapshot memory image layout. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ResidentMemoryRange { + /// Base host virtual address where this resident range currently resides. + pub base_host_virt_addr: u64, + /// Offset in the contiguous memory snapshot image. + pub image_offset: u64, + /// Length of the resident range in bytes. + pub length: u64, +} + +/// Resident memory ranges for a VM, using the same contiguous layout as memory snapshots. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct ResidentMemoryRanges { + /// Page size used by `mincore(2)`. + pub page_size: usize, + /// Total size of the contiguous memory snapshot image. + pub memory_size: u64, + /// Resident ranges coalesced across adjacent pages within each memory slot. + pub ranges: Vec, +} + /// Type of GuestRegionMmap. pub type GuestRegionMmap = vm_memory::GuestRegionMmap>; /// Type of GuestMemoryMmap. diff --git a/src/vmm/src/vstate/prefault.rs b/src/vmm/src/vstate/prefault.rs index 29c41c674..b694dcec9 100644 --- a/src/vmm/src/vstate/prefault.rs +++ b/src/vmm/src/vstate/prefault.rs @@ -5,6 +5,7 @@ use std::io; use std::mem::size_of; +use std::time::Instant; use kvm_bindings::kvm_pre_fault_memory; use kvm_ioctls::VcpuFd; @@ -39,6 +40,67 @@ pub struct PreFaultMemoryRequest { pub ranges: Vec, } +/// Completion statistics for the work executed by one vCPU worker. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +pub struct PreFaultMemoryWorkerStats { + /// Zero-based vCPU worker index. + pub vcpu_id: u32, + /// Number of input ranges assigned to this worker. + pub range_count: u64, + /// Bytes assigned to this worker. + pub requested_bytes: u64, + /// Bytes for which the kernel reported forward progress. + pub completed_bytes: u64, + /// Bytes remaining when the worker completed. Successful work must be zero. + pub remaining_bytes: u64, + /// Number of KVM_PRE_FAULT_MEMORY ioctl calls, including interrupted calls. + pub ioctl_count: u64, + /// Worker wall time in microseconds. + pub wall_time_us: u64, +} + +/// Completion statistics returned by the pre-fault-memory API. +#[derive(Clone, Debug, Default, Eq, PartialEq, Serialize)] +pub struct PreFaultMemoryStats { + /// Number of input ranges in the request. + pub range_count: u64, + /// Total bytes requested by the caller. + pub requested_bytes: u64, + /// Total bytes completed by all workers. + pub completed_bytes: u64, + /// Total bytes remaining across all workers. Successful work must be zero. + pub remaining_bytes: u64, + /// Total KVM_PRE_FAULT_MEMORY ioctl calls across all workers. + pub ioctl_count: u64, + /// End-to-end VMM wall time in microseconds, including worker dispatch and join. + pub wall_time_us: u64, + /// Per-worker completion statistics. + pub workers: Vec, +} + +impl PreFaultMemoryStats { + /// Builds aggregate statistics after every worker has reported completion. + pub fn from_workers( + ranges: &[PreFaultMemoryRange], + workers: Vec, + wall_time_us: u64, + ) -> Self { + let requested_bytes = ranges.iter().map(|range| range.size).sum(); + let completed_bytes = workers.iter().map(|worker| worker.completed_bytes).sum(); + let remaining_bytes = workers.iter().map(|worker| worker.remaining_bytes).sum(); + let ioctl_count = workers.iter().map(|worker| worker.ioctl_count).sum(); + Self { + range_count: u64::try_from(ranges.len()).expect("range count exceeds u64"), + requested_bytes, + completed_bytes, + remaining_bytes, + ioctl_count, + wall_time_us, + workers, + } + } +} + /// Structural validation failures for a pre-fault request. #[derive(Debug, Eq, PartialEq, thiserror::Error, displaydoc::Display)] pub enum PreFaultMemoryValidationError { @@ -205,34 +267,38 @@ pub fn split_pre_fault_ranges( pub(crate) fn send_pre_fault_events( work: &[Vec], mut send: F, -) -> Option<(usize, VcpuSendEventError)> +) -> (Option<(usize, VcpuSendEventError)>, Vec) where F: FnMut(usize, Vec) -> Result<(), VcpuSendEventError>, { let mut first_error = None; + let mut successful_vcpu_ids = Vec::with_capacity(work.len()); for (vcpu_id, ranges) in work.iter().enumerate() { - // send_event enqueues before signaling, so even a signal error has a response to drain. - if let Err(error) = send(vcpu_id, ranges.clone()) - && first_error.is_none() - { - first_error = Some((vcpu_id, error)); + match send(vcpu_id, ranges.clone()) { + Ok(()) => successful_vcpu_ids.push(vcpu_id), + Err(error) if first_error.is_none() => first_error = Some((vcpu_id, error)), + Err(_) => {} } } - first_error + (first_error, successful_vcpu_ids) } -/// Receives every response for a pre-fault operation and returns the first response error. +/// Receives responses only from workers whose event signal succeeded. pub(crate) fn drain_pre_fault_responses( - worker_count: usize, + worker_ids: &[usize], mut receive: F, -) -> Option +) -> Result, PreFaultMemoryError> where F: FnMut(usize) -> Result, { let mut first_error = None; - for vcpu_id in 0..worker_count { + let mut worker_stats = Vec::with_capacity(worker_ids.len()); + for &vcpu_id in worker_ids { let response_error = match receive(vcpu_id) { - Ok(VcpuResponse::PreFaultMemoryCompleted) => None, + Ok(VcpuResponse::PreFaultMemoryCompleted(stats)) => { + worker_stats.push(stats); + None + } Ok(VcpuResponse::Error(source)) => Some(PreFaultMemoryError::Vcpu { vcpu_id, source }), Ok(VcpuResponse::NotAllowed(reason)) => { Some(PreFaultMemoryError::UnexpectedResponse { vcpu_id, reason }) @@ -254,15 +320,18 @@ where first_error = response_error; } } - first_error + first_error.map_or(Ok(worker_stats), Err) } /// Executes pre-fault ioctls for a single vCPU's work queue. pub fn pre_fault_memory( vcpu_fd: &VcpuFd, ranges: &[PreFaultMemoryRange], -) -> Result<(), PreFaultMemoryIoctlError> { - run_pre_fault_memory(ranges, |request| { + vcpu_id: u32, +) -> Result { + let started = Instant::now(); + let (requested_bytes, completed_bytes, remaining_bytes, ioctl_count) = + run_pre_fault_memory(ranges, |request| { // SAFETY: The caller invokes this function from the owning vCPU thread with a valid // KVM vCPU fd. `kvm_pre_fault_memory` is the bindgen representation of the UAPI // struct, exactly 64 bytes, and remains alive and exclusively borrowed for the @@ -273,9 +342,22 @@ pub fn pre_fault_memory( } else { Ok(()) } + })?; + Ok(PreFaultMemoryWorkerStats { + vcpu_id, + range_count: u64::try_from(ranges.len()).expect("range count exceeds u64"), + requested_bytes, + completed_bytes, + remaining_bytes, + ioctl_count, + wall_time_us: elapsed_us(started), }) } +fn elapsed_us(started: Instant) -> u64 { + u64::try_from(started.elapsed().as_micros()).unwrap_or(u64::MAX) +} + const KVMIO: ::std::os::raw::c_uint = 0xAE; mod pre_fault_memory_ioctl { @@ -291,10 +373,13 @@ use pre_fault_memory_ioctl::KVM_PRE_FAULT_MEMORY; fn run_pre_fault_memory( ranges: &[PreFaultMemoryRange], mut ioctl: F, -) -> Result<(), PreFaultMemoryIoctlError> +) -> Result<(u64, u64, u64, u64), PreFaultMemoryIoctlError> where F: FnMut(&mut kvm_pre_fault_memory) -> io::Result<()>, { + let requested_bytes: u64 = ranges.iter().map(|range| range.size).sum(); + let mut completed_bytes = 0; + let mut ioctl_count = 0; for range in ranges { let mut request = kvm_pre_fault_memory { gpa: range.gpa, @@ -309,6 +394,7 @@ where request.flags = 0; request.padding = [0; 5]; + ioctl_count += 1; match ioctl(&mut request) { Err(error) if error.raw_os_error() == Some(libc::EINTR) => { request.gpa = previous_gpa; @@ -327,12 +413,15 @@ where size: request.size, }); } - Ok(()) => {} + Ok(()) => completed_bytes += previous_size - request.size, } } } - Ok(()) + let remaining_bytes = requested_bytes + .checked_sub(completed_bytes) + .expect("pre-fault completion cannot exceed bytes requested"); + Ok((requested_bytes, completed_bytes, remaining_bytes, ioctl_count)) } #[cfg(test)] @@ -347,6 +436,13 @@ mod tests { PreFaultMemoryRange { gpa, size } } + fn worker_stats(vcpu_id: u32) -> PreFaultMemoryWorkerStats { + PreFaultMemoryWorkerStats { + vcpu_id, + ..Default::default() + } + } + #[test] fn validates_ranges_and_totals() { let cases = [ @@ -441,7 +537,7 @@ mod tests { } #[test] - fn signal_failure_is_recorded_and_all_workers_are_drained() { + fn signal_failure_skips_unresponsive_worker() { let work = vec![ vec![range(0, 0x1000)], vec![range(0x1000, 0x1000)], @@ -449,7 +545,7 @@ mod tests { ]; let mut sent = Vec::new(); let mut receiving = false; - let send_error = send_pre_fault_events(&work, |vcpu_id, _ranges| { + let (send_error, successful_vcpu_ids) = send_pre_fault_events(&work, |vcpu_id, _ranges| { assert!(!receiving); sent.push(vcpu_id); if vcpu_id == 1 { @@ -462,21 +558,21 @@ mod tests { }); assert_eq!(sent, vec![0, 1, 2]); assert_eq!(send_error.as_ref().map(|(vcpu_id, _)| *vcpu_id), Some(1)); + assert_eq!(successful_vcpu_ids, vec![0, 2]); let mut first_error = send_error.map(|(vcpu_id, source)| PreFaultMemoryError::VcpuSend { vcpu_id, source }); let mut responses = VecDeque::from([ - VcpuResponse::PreFaultMemoryCompleted, - VcpuResponse::PreFaultMemoryCompleted, - VcpuResponse::PreFaultMemoryCompleted, + VcpuResponse::PreFaultMemoryCompleted(worker_stats(0)), + VcpuResponse::PreFaultMemoryCompleted(worker_stats(2)), ]); - let response_error = drain_pre_fault_responses(work.len(), |_vcpu_id| { + let response_error = drain_pre_fault_responses(&successful_vcpu_ids, |_vcpu_id| { receiving = true; assert!(receiving); Ok(responses.pop_front().expect("missing test response")) }); if first_error.is_none() { - first_error = response_error; + first_error = response_error.err(); } assert!(matches!( @@ -490,17 +586,17 @@ mod tests { fn response_failure_does_not_skip_other_workers() { let mut responses = VecDeque::from([ VcpuResponse::Error(VcpuError::FaultyKvmExit("worker failure".to_string())), - VcpuResponse::PreFaultMemoryCompleted, - VcpuResponse::PreFaultMemoryCompleted, + VcpuResponse::PreFaultMemoryCompleted(worker_stats(1)), + VcpuResponse::PreFaultMemoryCompleted(worker_stats(2)), ]); - let error = drain_pre_fault_responses(3, |_vcpu_id| { + let error = drain_pre_fault_responses(&[0, 1, 2], |_vcpu_id| { Ok(responses.pop_front().expect("missing test response")) }); assert!(matches!( error, - Some(PreFaultMemoryError::Vcpu { vcpu_id: 0, .. }) + Err(PreFaultMemoryError::Vcpu { vcpu_id: 0, .. }) )); assert!(responses.is_empty()); } @@ -512,20 +608,20 @@ mod tests { let worker = thread::spawn(move || { release_receiver.recv().unwrap(); response_sender - .send(VcpuResponse::PreFaultMemoryCompleted) + .send(VcpuResponse::PreFaultMemoryCompleted(worker_stats(0))) .unwrap(); response_sender.send(VcpuResponse::Resumed).unwrap(); }); let mut first_receive = true; - let error = drain_pre_fault_responses(1, |_vcpu_id| { + let error = drain_pre_fault_responses(&[0], |_vcpu_id| { assert!(first_receive); first_receive = false; release_sender.send(()).unwrap(); response_receiver.recv() }); - assert!(error.is_none()); + assert!(error.is_ok()); assert!(matches!( response_receiver.recv().unwrap(), VcpuResponse::Resumed @@ -561,7 +657,7 @@ mod tests { fn ioctl_loop_handles_partial_progress_and_eintr() { let ranges = [range(0x1000, 0x3000)]; let mut calls = 0; - run_pre_fault_memory(&ranges, |request| { + let stats = run_pre_fault_memory(&ranges, |request| { calls += 1; assert_eq!(request.flags, 0); assert_eq!(request.padding, [0; 5]); @@ -578,6 +674,7 @@ mod tests { }) .unwrap(); assert_eq!(calls, 4); + assert_eq!(stats, (0x3000, 0x3000, 0, 4)); } #[test] diff --git a/src/vmm/src/vstate/vcpu.rs b/src/vmm/src/vstate/vcpu.rs index d2bdaddf9..6e4712bb2 100644 --- a/src/vmm/src/vstate/vcpu.rs +++ b/src/vmm/src/vstate/vcpu.rs @@ -29,7 +29,9 @@ use crate::utils::signal::{Killable, register_signal_handler, sigrtmin}; use crate::utils::sm::StateMachine; use crate::vstate::bus::Bus; use crate::vstate::prefault::pre_fault_memory; -use crate::vstate::prefault::{PreFaultMemoryIoctlError, PreFaultMemoryRange}; +use crate::vstate::prefault::{ + PreFaultMemoryIoctlError, PreFaultMemoryRange, PreFaultMemoryWorkerStats, +}; use crate::vstate::vm::Vm; /// Signal number (SIGRTMIN) used to kick Vcpus. @@ -286,7 +288,7 @@ impl Vcpu { .expect("vcpu channel unexpectedly closed"); } // Pre-faulting cannot be performed on a running Vcpu. - Ok(VcpuEvent::PreFaultMemory(_)) => { + Ok(VcpuEvent::PreFaultMemory { .. }) => { self.response_sender .send(VcpuResponse::NotAllowed(String::from( "pre-fault memory requires a paused vCPU", @@ -363,11 +365,11 @@ impl Vcpu { StateMachine::next(Self::paused) } - Ok(VcpuEvent::PreFaultMemory(ranges)) => { - pre_fault_memory(&self.kvm_vcpu.fd, &ranges) - .map(|()| { + Ok(VcpuEvent::PreFaultMemory { ranges }) => { + pre_fault_memory(&self.kvm_vcpu.fd, &ranges, u32::from(self.kvm_vcpu.index)) + .map(|stats| { self.response_sender - .send(VcpuResponse::PreFaultMemoryCompleted) + .send(VcpuResponse::PreFaultMemoryCompleted(stats)) .expect("vcpu channel unexpectedly closed"); }) .unwrap_or_else(|err| { @@ -549,7 +551,10 @@ pub enum VcpuEvent { /// Event to dump CPU configuration of a paused Vcpu. DumpCpuConfig, /// Event to pre-fault selected memory of a paused Vcpu. - PreFaultMemory(Vec), + PreFaultMemory { + /// Work assigned to this vCPU. + ranges: Vec, + }, } /// List of responses that the Vcpu reports. @@ -568,8 +573,8 @@ pub enum VcpuResponse { SavedState(Box), /// Vcpu is in the state where CPU config is dumped. DumpedCpuConfig(Box), - /// Requested memory pre-faulting completed. - PreFaultMemoryCompleted, + /// Requested memory pre-faulting completed, with per-worker statistics. + PreFaultMemoryCompleted(PreFaultMemoryWorkerStats), } impl fmt::Debug for VcpuResponse { @@ -583,7 +588,9 @@ impl fmt::Debug for VcpuResponse { Error(err) => write!(f, "VcpuResponse::Error({:?})", err), NotAllowed(reason) => write!(f, "VcpuResponse::NotAllowed({})", reason), DumpedCpuConfig(_) => write!(f, "VcpuResponse::DumpedCpuConfig"), - PreFaultMemoryCompleted => write!(f, "VcpuResponse::PreFaultMemoryCompleted"), + PreFaultMemoryCompleted(stats) => { + write!(f, "VcpuResponse::PreFaultMemoryCompleted({stats:?})") + } } } } @@ -845,7 +852,7 @@ pub(crate) mod tests { | NotAllowed(_) | SavedState(_) | DumpedCpuConfig(_) - | PreFaultMemoryCompleted => (), + | PreFaultMemoryCompleted(_) => (), }; match (self, other) { (Paused, Paused) | (Resumed, Resumed) => true, @@ -853,7 +860,7 @@ pub(crate) mod tests { (NotAllowed(_), NotAllowed(_)) | (SavedState(_), SavedState(_)) | (DumpedCpuConfig(_), DumpedCpuConfig(_)) - | (PreFaultMemoryCompleted, PreFaultMemoryCompleted) => true, + | (PreFaultMemoryCompleted(_), PreFaultMemoryCompleted(_)) => true, (Error(err), Error(other_err)) => { format!("{:?}", err) == format!("{:?}", other_err) } diff --git a/src/vmm/src/vstate/vm.rs b/src/vmm/src/vstate/vm.rs index 149405dc0..97a98259e 100644 --- a/src/vmm/src/vstate/vm.rs +++ b/src/vmm/src/vstate/vm.rs @@ -35,7 +35,8 @@ use crate::vstate::bus::Bus; use crate::vstate::interrupts::{InterruptError, MsixVector, MsixVectorConfig, MsixVectorGroup}; use crate::vstate::memory::{ DirtyMemoryRanges, GuestMemory, GuestMemoryExtension, GuestMemoryMmap, GuestMemoryRegion, - GuestMemoryState, GuestRegionMmap, GuestRegionMmapExt, MemoryError, + GuestMemoryState, GuestRegionMmap, GuestRegionMmapExt, MemoryError, ResidentMemoryRange, + ResidentMemoryRanges, }; use crate::vstate::resources::ResourceAllocator; use crate::vstate::vcpu::VcpuError; @@ -341,6 +342,44 @@ impl Vm { .map_err(VmError::MemoryError) } + /// Returns host-resident guest-memory ranges according to `mincore(2)`. + /// + /// This is intentionally independent from KVM dirty-page tracking and does + /// not read or modify any dirty bitmap state. + pub fn get_resident_memory_ranges(&self) -> Result { + let resident_bitmap = self + .guest_memory() + .iter() + .flat_map(|region| region.plugged_slots()) + .map(|mem_slot| { + mincore_bitmap( + mem_slot.slice.ptr_guard_mut().as_ptr(), + mem_slot.slice.len(), + ) + .map(|bitmap| (mem_slot.slot, bitmap)) + }) + .collect::>()?; + let page_size = get_page_size().map_err(MemoryError::PageSize)?; + let ranges = self + .guest_memory() + .dirty_memory_ranges(&resident_bitmap, page_size) + .map_err(VmError::MemoryError)?; + + Ok(ResidentMemoryRanges { + page_size: ranges.page_size, + memory_size: ranges.memory_size, + ranges: ranges + .ranges + .into_iter() + .map(|range| ResidentMemoryRange { + base_host_virt_addr: range.base_host_virt_addr, + image_offset: range.image_offset, + length: range.length, + }) + .collect(), + }) + } + /// Takes a snapshot of the virtual machine running inside the given [`Vmm`] and saves it to /// `mem_file_path`. /// diff --git a/src/vmm/tests/integration_tests.rs b/src/vmm/tests/integration_tests.rs index a9117617c..49ef42006 100644 --- a/src/vmm/tests/integration_tests.rs +++ b/src/vmm/tests/integration_tests.rs @@ -549,10 +549,17 @@ fn assert_pre_fault_supported_or_host_unsupported(result: Result { + assert_eq!(stats.requested_bytes, stats.completed_bytes); + assert_eq!(stats.remaining_bytes, 0); + assert!(!stats.workers.is_empty()); + } + _ => assert!( + mode_unsupported, + "KVM_CAP_PRE_FAULT_MEMORY={capability}, expected completion stats or EOPNOTSUPP for the current vCPU mode, got {result:?}" + ), + } } else { assert!( matches!(