ostd/vm/gpm_space.rs
1// SPDX-License-Identifier: MPL-2.0
2
3//! Guest physical memory space.
4
5use core::{mem::ManuallyDrop, ops::Range};
6
7use super::Gpaddr;
8use crate::{
9 arch::vm::memory::{ArchGuestPhysMemSpace, PageTableEntry, PagingConsts},
10 mm::{
11 AnyUFrameMeta, Frame, PageFlags, PageProperty, PagingLevel, UFrame,
12 frame::FrameRef,
13 page_table::{self, PageTable, PageTableConfig, PageTableFrag},
14 },
15 prelude::*,
16 smp::PendingIpis,
17 sync::RcuDrop,
18 task::atomic_mode::AsAtomicModeGuard,
19};
20
21/// Guest physical memory space of a VM.
22///
23/// This type owns the page table that maps guest physical addresses to
24/// host physical frames. Its cursors manage 4-KiB mappings within a 48-bit
25/// guest physical address space. Each mapping must permit reads; write and
26/// execute permissions can be added independently.
27///
28/// This address space must be dropped outside [atomic mode](crate::task::atomic_mode).
29pub struct GuestPhysMemSpace {
30 pt: PageTable<GuestPtConfig>,
31 arch: ArchGuestPhysMemSpace,
32}
33
34impl GuestPhysMemSpace {
35 /// Creates a new guest physical memory space.
36 ///
37 /// # Errors
38 ///
39 /// Returns an error if the CPU does not support second-stage address
40 /// translation or the required invalidation operations.
41 ///
42 /// # Panics
43 ///
44 /// Panics if called in [atomic mode](crate::task::atomic_mode).
45 pub fn new() -> Result<Self> {
46 let arch = ArchGuestPhysMemSpace::new()?;
47 let pt = PageTable::<GuestPtConfig>::empty();
48
49 // A reused root-table address may still have cached translations.
50 arch.invalidate(&[]).wait();
51
52 Ok(Self { pt, arch })
53 }
54
55 /// Gets an immutable cursor over a guest physical address range.
56 ///
57 /// The cursor behaves like a lock guard, exclusively owning a sub-tree of
58 /// the page table, preventing others from creating a cursor in it. So be
59 /// sure to drop the cursor as soon as possible.
60 ///
61 /// The creation of the cursor may block if another cursor having an
62 /// overlapping range is alive.
63 pub fn cursor<'a, G: AsAtomicModeGuard>(
64 &'a self,
65 guard: &'a G,
66 gpa: &Range<Gpaddr>,
67 ) -> Result<Cursor<'a>> {
68 Ok(Cursor(self.pt.cursor(guard, gpa)?))
69 }
70
71 /// Gets a mutable cursor over a guest physical address range.
72 ///
73 /// The same as [`Self::cursor`], the cursor behaves like a lock guard,
74 /// exclusively owning a sub-tree of the page table, preventing others
75 /// from creating a cursor in it. So be sure to drop the cursor as soon as
76 /// possible.
77 ///
78 /// The creation of the cursor may block if another cursor having an
79 /// overlapping range is alive.
80 pub fn cursor_mut<'a, G: AsAtomicModeGuard>(
81 &'a self,
82 guard: &'a G,
83 gpa: &Range<Gpaddr>,
84 ) -> Result<CursorMut<'a>> {
85 Ok(CursorMut {
86 pt_cursor: self.pt.cursor_mut(guard, gpa)?,
87 arch: &self.arch,
88 pending_ipis: PendingIpis::new_empty(),
89 })
90 }
91
92 /// Returns the page table's root physical address for this
93 /// guest memory space.
94 ///
95 /// The caller must keep this address space borrowed while
96 /// its page table is in use.
97 pub(crate) fn root_paddr(&self) -> u64 {
98 self.pt.root_paddr() as u64
99 }
100}
101
102/// A borrowed backing frame and its page properties.
103pub type QueriedItem<'a> = (FrameRef<'a, dyn AnyUFrameMeta>, PageProperty);
104
105/// The cursor for querying over the guest physical memory space without modifying it.
106///
107/// It exclusively owns a sub-tree of the page table, preventing others from
108/// reading or modifying the same sub-tree. Two read-only cursors can not be
109/// created from the same guest physical address range either.
110pub struct Cursor<'a>(page_table::Cursor<'a, GuestPtConfig>);
111
112impl Cursor<'_> {
113 /// Queries the mapping at the current guest physical address.
114 ///
115 /// If the cursor is pointing to a valid guest physical address that is
116 /// locked, it will return the guest physical address range, the borrowed
117 /// backing frame, and its page properties.
118 pub fn query(&mut self) -> Result<(Range<Gpaddr>, Option<QueriedItem<'_>>)> {
119 Ok(self.0.query()?)
120 }
121
122 /// Moves the cursor forward to the next mapped guest physical address.
123 ///
124 /// If there is a mapped guest physical address following the current
125 /// address within next `len` bytes, it will return that mapped address. In
126 /// this case, the cursor will stop at the mapped address.
127 ///
128 /// Otherwise, it will return `None`. And the cursor may stop at any
129 /// address after `len` bytes.
130 ///
131 /// # Panics
132 ///
133 /// Panics if:
134 /// - the length is longer than the remaining range of the cursor;
135 /// - the length is not page-aligned.
136 pub fn find_next(&mut self, len: usize) -> Option<Gpaddr> {
137 self.0.find_next(len)
138 }
139
140 /// Jumps to the guest physical address.
141 ///
142 /// If the target address is out of the range, this method will return `Err`.
143 ///
144 /// # Panics
145 ///
146 /// This method panics if the address has bad alignment.
147 pub fn jump(&mut self, gpa: Gpaddr) -> Result<()> {
148 self.0.jump(gpa)?;
149 Ok(())
150 }
151
152 /// Gets the guest physical address of the current slot.
153 pub fn gpa(&self) -> Gpaddr {
154 self.0.virt_addr()
155 }
156}
157
158/// The cursor for modifying the mappings in guest physical memory space.
159///
160/// It exclusively owns a sub-tree of the page table, preventing others from
161/// reading or modifying the same sub-tree.
162pub struct CursorMut<'a> {
163 pt_cursor: page_table::CursorMut<'a, GuestPtConfig>,
164 arch: &'a ArchGuestPhysMemSpace,
165 pending_ipis: PendingIpis,
166}
167
168impl<'a> CursorMut<'a> {
169 /// Queries the mapping at the current guest physical address.
170 ///
171 /// This is the same as [`Cursor::query`].
172 ///
173 /// If the cursor is pointing to a valid guest physical address that is
174 /// locked, it will return the guest physical address range, the borrowed
175 /// backing frame, and its page properties.
176 pub fn query(&mut self) -> Result<(Range<Gpaddr>, Option<QueriedItem<'_>>)> {
177 Ok(self.pt_cursor.query()?)
178 }
179
180 /// Moves the cursor forward to the next mapped guest physical address.
181 ///
182 /// This is the same as [`Cursor::find_next`].
183 pub fn find_next(&mut self, len: usize) -> Option<Gpaddr> {
184 self.pt_cursor.find_next(len)
185 }
186
187 /// Jumps to the guest physical address.
188 ///
189 /// This is the same as [`Cursor::jump`].
190 ///
191 /// # Panics
192 ///
193 /// This method panics if the address has bad alignment.
194 pub fn jump(&mut self, gpa: Gpaddr) -> Result<()> {
195 self.pt_cursor.jump(gpa)?;
196 Ok(())
197 }
198
199 /// Gets the guest physical address of the current slot.
200 pub fn gpa(&self) -> Gpaddr {
201 self.pt_cursor.virt_addr()
202 }
203
204 /// Maps a frame into the current slot.
205 ///
206 /// This method will bring the cursor to the next slot after the modification.
207 ///
208 /// # Panics
209 ///
210 /// Panics if:
211 /// - the current guest physical address is already mapped;
212 /// - the current guest physical address is outside the cursor's range.
213 pub fn map(&mut self, frame: UFrame, prop: PageProperty) {
214 let item: GuestPtItem = (frame, prop);
215
216 // SAFETY: It is safe to map untyped memory into guest physical memory.
217 unsafe { self.pt_cursor.map(item) };
218 }
219
220 /// Applies the operation to the next slot of mapping within the range.
221 ///
222 /// The range to be found in is the current guest physical address with the
223 /// provided length.
224 ///
225 /// The function stops and yields the actually protected range if it has
226 /// actually protected a page, no matter if the following pages are also
227 /// required to be protected.
228 ///
229 /// It also makes the cursor moves forward to the next page after the
230 /// protected one. If no mapped pages exist in the following range, the
231 /// cursor will stop at the end of the range and return [`None`].
232 ///
233 /// Cached translations are invalidated on the current CPU before returning,
234 /// and asynchronously on remote CPUs. Use [`Self::sync_tlb_flush`] to
235 /// wait for remote invalidations to complete.
236 ///
237 /// # Panics
238 ///
239 /// Panics if:
240 /// - the length is longer than the remaining range of the cursor;
241 /// - the length is not page-aligned.
242 pub fn protect_next(
243 &mut self,
244 len: usize,
245 op: &mut impl FnMut(&mut PageFlags),
246 ) -> Option<Range<Gpaddr>> {
247 // SAFETY: It is safe to set `PageFlags` of guest physical memory.
248 let range = unsafe {
249 self.pt_cursor
250 .protect_next(len, &mut |prop| op(&mut prop.flags))
251 }?;
252 self.pending_ipis.extend(&self.arch.invalidate(&[]));
253 Some(range)
254 }
255
256 /// Clears the mapping starting from the current slot,
257 /// and returns the number of unmapped pages.
258 ///
259 /// This method brings the cursor forward by at least `len` bytes in the
260 /// guest physical address space, but not past the end of the cursor's range.
261 ///
262 /// Already-absent mappings encountered by the cursor will be skipped.
263 /// It is valid to unmap a range that is not mapped.
264 ///
265 /// This method issues and dispatches EPT invalidation for removed mappings.
266 /// Cached translations are invalidated on the current CPU before returning,
267 /// and asynchronously on remote CPUs. Removed frames are retained until
268 /// invalidation completes on all CPUs. Use [`Self::sync_tlb_flush`] to
269 /// wait for remote invalidations to complete. Using a large `len` avoids
270 /// the overhead of multiple invalidations.
271 ///
272 /// # Panics
273 ///
274 /// Panics if:
275 /// - the length is longer than the remaining range of the cursor;
276 /// - the length is not page-aligned.
277 pub fn unmap(&mut self, len: usize) -> usize {
278 let end_gpa = self.gpa() + len;
279 let mut num_unmapped: usize = 0;
280 // Retain removed frames even if unwinding happens before dispatch.
281 let mut frames = ManuallyDrop::new(Vec::new());
282 loop {
283 frames.reserve(1);
284 // SAFETY:
285 // 1. It is safe to unmap guest physical memory.
286 // 2. Removed frames are retained below, then cloned into each CPU's
287 // invalidation queue before their references here are released.
288 let Some(frag) = (unsafe { self.pt_cursor.take_next(end_gpa - self.gpa()) }) else {
289 break; // No more mappings in the range.
290 };
291
292 match frag {
293 PageTableFrag::Mapped { item, .. } => {
294 // SAFETY: The frame will be dropped after the RCU grace
295 // period (see the following `RcuDrop::new`).
296 let ((frame, _), panic_guard) = unsafe { RcuDrop::into_inner(item) };
297 frames.push(Frame::rcu_from_unsized(RcuDrop::new(frame)));
298 panic_guard.forget();
299 num_unmapped += 1;
300 }
301 PageTableFrag::StrayPageTable { pt, num_frames, .. } => {
302 frames.push(Frame::rcu_from_unsized(pt));
303 num_unmapped += num_frames;
304 }
305 }
306 }
307
308 if !frames.is_empty() {
309 self.pending_ipis.extend(&self.arch.invalidate(&frames));
310 }
311 drop(ManuallyDrop::into_inner(frames));
312 num_unmapped
313 }
314
315 /// Waits for this cursor's previous EPT invalidations to complete on all CPUs.
316 ///
317 /// This synchronizes invalidations issued by [`Self::unmap`] and
318 /// [`Self::protect_next`]. Dropping the cursor does not wait for completion;
319 /// removed frames are retained until invalidation completes regardless.
320 ///
321 /// # Panics
322 ///
323 /// Panics if local IRQs are disabled.
324 pub fn sync_tlb_flush(&mut self) {
325 self.pending_ipis.wait();
326 self.pending_ipis = PendingIpis::new_empty();
327 }
328}
329
330/// A page table mapping guest physical addresses to host physical frames.
331#[derive(Clone, Debug)]
332pub(crate) struct GuestPtConfig {}
333
334// SAFETY: `item_raw_info`, `item_into_raw`, `item_from_raw`, and
335// `item_ref_from_raw` are correctly implemented with respect to the `Item` and
336// `ItemRef` types.
337unsafe impl PageTableConfig for GuestPtConfig {
338 const TOP_LEVEL_INDEX_RANGE: Range<usize> = 0..512;
339
340 type E = PageTableEntry;
341 type C = PagingConsts;
342
343 /// All mappings are tracked untyped frames.
344 type Item = GuestPtItem;
345 type ItemRef<'a> = GuestPtItemRef<'a>;
346
347 fn item_raw_info(item: &Self::Item) -> (Paddr, PagingLevel, PageProperty) {
348 let (frame, prop) = item;
349 (frame.paddr(), frame.map_level(), *prop)
350 }
351
352 unsafe fn item_from_raw(paddr: Paddr, level: PagingLevel, prop: PageProperty) -> Self::Item {
353 debug_assert_eq!(level, 1);
354 // SAFETY: The caller ensures safety.
355 let frame = unsafe { Frame::<dyn AnyUFrameMeta>::from_raw(paddr) };
356 (frame, prop)
357 }
358
359 unsafe fn item_ref_from_raw<'a>(
360 paddr: Paddr,
361 level: PagingLevel,
362 prop: PageProperty,
363 ) -> Self::ItemRef<'a> {
364 debug_assert_eq!(level, 1);
365 // SAFETY: The caller ensures that the frame outlives `'a` and that
366 // the type matches the frame.
367 let frame = unsafe { FrameRef::<dyn AnyUFrameMeta>::borrow_paddr(paddr) };
368 (frame, prop)
369 }
370}
371
372pub(crate) type GuestPtItem = (UFrame, PageProperty);
373pub(crate) type GuestPtItemRef<'a> = (FrameRef<'a, dyn AnyUFrameMeta>, PageProperty);
374
375#[cfg(ktest)]
376mod test {
377 use super::*;
378 use crate::{
379 Error,
380 mm::{CachePolicy, FrameAllocOptions, PAGE_SIZE},
381 task::disable_preempt,
382 };
383
384 #[ktest]
385 fn guest_mapping_lifecycle() {
386 let space = match GuestPhysMemSpace::new() {
387 Ok(space) => space,
388 Err(Error::NotEnoughResources | Error::AccessDenied) => {
389 crate::early_print!(" [skipped: VMX/EPT invalidation unavailable]");
390 return;
391 }
392 Err(err) => panic!("failed to create guest physical memory: {:?}", err),
393 };
394
395 let first = FrameAllocOptions::new().alloc_frame().unwrap();
396 let second = FrameAllocOptions::new().alloc_frame().unwrap();
397 let first_paddr = first.paddr();
398 let second_paddr = second.paddr();
399 let prop = PageProperty::new_guest(PageFlags::RWX, CachePolicy::Writeback);
400 let range = PAGE_SIZE..4 * PAGE_SIZE;
401
402 let guard = disable_preempt();
403 let mut cursor = space.cursor_mut(&guard, &range).unwrap();
404
405 // Test `map`.
406 cursor.map(first.into(), prop);
407 cursor.map(second.into(), prop);
408 assert_eq!(cursor.gpa(), 3 * PAGE_SIZE);
409
410 // Test `query`.
411 cursor.jump(2 * PAGE_SIZE).unwrap();
412 let (queried_range, item) = cursor.query().unwrap();
413 assert_eq!(queried_range, 2 * PAGE_SIZE..3 * PAGE_SIZE);
414 assert_eq!(
415 item.map(|(frame, prop)| (frame.paddr(), prop)),
416 Some((second_paddr, prop))
417 );
418 cursor.jump(PAGE_SIZE).unwrap();
419 let (queried_range, item) = cursor.query().unwrap();
420 assert_eq!(queried_range, PAGE_SIZE..2 * PAGE_SIZE);
421 assert_eq!(
422 item.map(|(frame, prop)| (frame.paddr(), prop)),
423 Some((first_paddr, prop))
424 );
425
426 // Test `protect`.
427 assert_eq!(
428 cursor.protect_next(PAGE_SIZE, &mut |flags| *flags = PageFlags::RX),
429 Some(PAGE_SIZE..2 * PAGE_SIZE)
430 );
431 cursor.jump(PAGE_SIZE).unwrap();
432 assert_eq!(cursor.query().unwrap().1.unwrap().1.flags, PageFlags::RX);
433
434 // Test `unmap`.
435 assert_eq!(cursor.unmap(2 * PAGE_SIZE), 2);
436 cursor.jump(PAGE_SIZE).unwrap();
437 assert!(cursor.find_next(3 * PAGE_SIZE).is_none());
438 }
439}