Skip to main content

ostd/vm/
gpm_space.rs

1// SPDX-License-Identifier: MPL-2.0
2
3//! Guest physical memory space.
4
5use core::{mem::ManuallyDrop, ops::Range};
6
7use super::Gpaddr;
8use crate::{
9    arch::vm::memory::{ArchGuestPhysMemSpace, PageTableEntry, PagingConsts},
10    mm::{
11        AnyUFrameMeta, Frame, PageFlags, PageProperty, PagingLevel, UFrame,
12        frame::FrameRef,
13        page_table::{self, PageTable, PageTableConfig, PageTableFrag},
14    },
15    prelude::*,
16    smp::PendingIpis,
17    sync::RcuDrop,
18    task::atomic_mode::AsAtomicModeGuard,
19};
20
21/// Guest physical memory space of a VM.
22///
23/// This type owns the page table that maps guest physical addresses to
24/// host physical frames. Its cursors manage 4-KiB mappings within a 48-bit
25/// guest physical address space. Each mapping must permit reads; write and
26/// execute permissions can be added independently.
27///
28/// This address space must be dropped outside [atomic mode](crate::task::atomic_mode).
29pub struct GuestPhysMemSpace {
30    pt: PageTable<GuestPtConfig>,
31    arch: ArchGuestPhysMemSpace,
32}
33
34impl GuestPhysMemSpace {
35    /// Creates a new guest physical memory space.
36    ///
37    /// # Errors
38    ///
39    /// Returns an error if the CPU does not support second-stage address
40    /// translation or the required invalidation operations.
41    ///
42    /// # Panics
43    ///
44    /// Panics if called in [atomic mode](crate::task::atomic_mode).
45    pub fn new() -> Result<Self> {
46        let arch = ArchGuestPhysMemSpace::new()?;
47        let pt = PageTable::<GuestPtConfig>::empty();
48
49        // A reused root-table address may still have cached translations.
50        arch.invalidate(&[]).wait();
51
52        Ok(Self { pt, arch })
53    }
54
55    /// Gets an immutable cursor over a guest physical address range.
56    ///
57    /// The cursor behaves like a lock guard, exclusively owning a sub-tree of
58    /// the page table, preventing others from creating a cursor in it. So be
59    /// sure to drop the cursor as soon as possible.
60    ///
61    /// The creation of the cursor may block if another cursor having an
62    /// overlapping range is alive.
63    pub fn cursor<'a, G: AsAtomicModeGuard>(
64        &'a self,
65        guard: &'a G,
66        gpa: &Range<Gpaddr>,
67    ) -> Result<Cursor<'a>> {
68        Ok(Cursor(self.pt.cursor(guard, gpa)?))
69    }
70
71    /// Gets a mutable cursor over a guest physical address range.
72    ///
73    /// The same as [`Self::cursor`], the cursor behaves like a lock guard,
74    /// exclusively owning a sub-tree of the page table, preventing others
75    /// from creating a cursor in it. So be sure to drop the cursor as soon as
76    /// possible.
77    ///
78    /// The creation of the cursor may block if another cursor having an
79    /// overlapping range is alive.
80    pub fn cursor_mut<'a, G: AsAtomicModeGuard>(
81        &'a self,
82        guard: &'a G,
83        gpa: &Range<Gpaddr>,
84    ) -> Result<CursorMut<'a>> {
85        Ok(CursorMut {
86            pt_cursor: self.pt.cursor_mut(guard, gpa)?,
87            arch: &self.arch,
88            pending_ipis: PendingIpis::new_empty(),
89        })
90    }
91
92    /// Returns the page table's root physical address for this
93    /// guest memory space.
94    ///
95    /// The caller must keep this address space borrowed while
96    /// its page table is in use.
97    pub(crate) fn root_paddr(&self) -> u64 {
98        self.pt.root_paddr() as u64
99    }
100}
101
102/// A borrowed backing frame and its page properties.
103pub type QueriedItem<'a> = (FrameRef<'a, dyn AnyUFrameMeta>, PageProperty);
104
105/// The cursor for querying over the guest physical memory space without modifying it.
106///
107/// It exclusively owns a sub-tree of the page table, preventing others from
108/// reading or modifying the same sub-tree. Two read-only cursors can not be
109/// created from the same guest physical address range either.
110pub struct Cursor<'a>(page_table::Cursor<'a, GuestPtConfig>);
111
112impl Cursor<'_> {
113    /// Queries the mapping at the current guest physical address.
114    ///
115    /// If the cursor is pointing to a valid guest physical address that is
116    /// locked, it will return the guest physical address range, the borrowed
117    /// backing frame, and its page properties.
118    pub fn query(&mut self) -> Result<(Range<Gpaddr>, Option<QueriedItem<'_>>)> {
119        Ok(self.0.query()?)
120    }
121
122    /// Moves the cursor forward to the next mapped guest physical address.
123    ///
124    /// If there is a mapped guest physical address following the current
125    /// address within next `len` bytes, it will return that mapped address. In
126    /// this case, the cursor will stop at the mapped address.
127    ///
128    /// Otherwise, it will return `None`. And the cursor may stop at any
129    /// address after `len` bytes.
130    ///
131    /// # Panics
132    ///
133    /// Panics if:
134    ///  - the length is longer than the remaining range of the cursor;
135    ///  - the length is not page-aligned.
136    pub fn find_next(&mut self, len: usize) -> Option<Gpaddr> {
137        self.0.find_next(len)
138    }
139
140    /// Jumps to the guest physical address.
141    ///
142    /// If the target address is out of the range, this method will return `Err`.
143    ///
144    /// # Panics
145    ///
146    /// This method panics if the address has bad alignment.
147    pub fn jump(&mut self, gpa: Gpaddr) -> Result<()> {
148        self.0.jump(gpa)?;
149        Ok(())
150    }
151
152    /// Gets the guest physical address of the current slot.
153    pub fn gpa(&self) -> Gpaddr {
154        self.0.virt_addr()
155    }
156}
157
158/// The cursor for modifying the mappings in guest physical memory space.
159///
160/// It exclusively owns a sub-tree of the page table, preventing others from
161/// reading or modifying the same sub-tree.
162pub struct CursorMut<'a> {
163    pt_cursor: page_table::CursorMut<'a, GuestPtConfig>,
164    arch: &'a ArchGuestPhysMemSpace,
165    pending_ipis: PendingIpis,
166}
167
168impl<'a> CursorMut<'a> {
169    /// Queries the mapping at the current guest physical address.
170    ///
171    /// This is the same as [`Cursor::query`].
172    ///
173    /// If the cursor is pointing to a valid guest physical address that is
174    /// locked, it will return the guest physical address range, the borrowed
175    /// backing frame, and its page properties.
176    pub fn query(&mut self) -> Result<(Range<Gpaddr>, Option<QueriedItem<'_>>)> {
177        Ok(self.pt_cursor.query()?)
178    }
179
180    /// Moves the cursor forward to the next mapped guest physical address.
181    ///
182    /// This is the same as [`Cursor::find_next`].
183    pub fn find_next(&mut self, len: usize) -> Option<Gpaddr> {
184        self.pt_cursor.find_next(len)
185    }
186
187    /// Jumps to the guest physical address.
188    ///
189    /// This is the same as [`Cursor::jump`].
190    ///
191    /// # Panics
192    ///
193    /// This method panics if the address has bad alignment.
194    pub fn jump(&mut self, gpa: Gpaddr) -> Result<()> {
195        self.pt_cursor.jump(gpa)?;
196        Ok(())
197    }
198
199    /// Gets the guest physical address of the current slot.
200    pub fn gpa(&self) -> Gpaddr {
201        self.pt_cursor.virt_addr()
202    }
203
204    /// Maps a frame into the current slot.
205    ///
206    /// This method will bring the cursor to the next slot after the modification.
207    ///
208    /// # Panics
209    ///
210    /// Panics if:
211    ///  - the current guest physical address is already mapped;
212    ///  - the current guest physical address is outside the cursor's range.
213    pub fn map(&mut self, frame: UFrame, prop: PageProperty) {
214        let item: GuestPtItem = (frame, prop);
215
216        // SAFETY: It is safe to map untyped memory into guest physical memory.
217        unsafe { self.pt_cursor.map(item) };
218    }
219
220    /// Applies the operation to the next slot of mapping within the range.
221    ///
222    /// The range to be found in is the current guest physical address with the
223    /// provided length.
224    ///
225    /// The function stops and yields the actually protected range if it has
226    /// actually protected a page, no matter if the following pages are also
227    /// required to be protected.
228    ///
229    /// It also makes the cursor moves forward to the next page after the
230    /// protected one. If no mapped pages exist in the following range, the
231    /// cursor will stop at the end of the range and return [`None`].
232    ///
233    /// Cached translations are invalidated on the current CPU before returning,
234    /// and asynchronously on remote CPUs. Use [`Self::sync_tlb_flush`] to
235    /// wait for remote invalidations to complete.
236    ///
237    /// # Panics
238    ///
239    /// Panics if:
240    ///  - the length is longer than the remaining range of the cursor;
241    ///  - the length is not page-aligned.
242    pub fn protect_next(
243        &mut self,
244        len: usize,
245        op: &mut impl FnMut(&mut PageFlags),
246    ) -> Option<Range<Gpaddr>> {
247        // SAFETY: It is safe to set `PageFlags` of guest physical memory.
248        let range = unsafe {
249            self.pt_cursor
250                .protect_next(len, &mut |prop| op(&mut prop.flags))
251        }?;
252        self.pending_ipis.extend(&self.arch.invalidate(&[]));
253        Some(range)
254    }
255
256    /// Clears the mapping starting from the current slot,
257    /// and returns the number of unmapped pages.
258    ///
259    /// This method brings the cursor forward by at least `len` bytes in the
260    /// guest physical address space, but not past the end of the cursor's range.
261    ///
262    /// Already-absent mappings encountered by the cursor will be skipped.
263    /// It is valid to unmap a range that is not mapped.
264    ///
265    /// This method issues and dispatches EPT invalidation for removed mappings.
266    /// Cached translations are invalidated on the current CPU before returning,
267    /// and asynchronously on remote CPUs. Removed frames are retained until
268    /// invalidation completes on all CPUs. Use [`Self::sync_tlb_flush`] to
269    /// wait for remote invalidations to complete. Using a large `len` avoids
270    /// the overhead of multiple invalidations.
271    ///
272    /// # Panics
273    ///
274    /// Panics if:
275    ///  - the length is longer than the remaining range of the cursor;
276    ///  - the length is not page-aligned.
277    pub fn unmap(&mut self, len: usize) -> usize {
278        let end_gpa = self.gpa() + len;
279        let mut num_unmapped: usize = 0;
280        // Retain removed frames even if unwinding happens before dispatch.
281        let mut frames = ManuallyDrop::new(Vec::new());
282        loop {
283            frames.reserve(1);
284            // SAFETY:
285            // 1. It is safe to unmap guest physical memory.
286            // 2. Removed frames are retained below, then cloned into each CPU's
287            //    invalidation queue before their references here are released.
288            let Some(frag) = (unsafe { self.pt_cursor.take_next(end_gpa - self.gpa()) }) else {
289                break; // No more mappings in the range.
290            };
291
292            match frag {
293                PageTableFrag::Mapped { item, .. } => {
294                    // SAFETY: The frame will be dropped after the RCU grace
295                    // period (see the following `RcuDrop::new`).
296                    let ((frame, _), panic_guard) = unsafe { RcuDrop::into_inner(item) };
297                    frames.push(Frame::rcu_from_unsized(RcuDrop::new(frame)));
298                    panic_guard.forget();
299                    num_unmapped += 1;
300                }
301                PageTableFrag::StrayPageTable { pt, num_frames, .. } => {
302                    frames.push(Frame::rcu_from_unsized(pt));
303                    num_unmapped += num_frames;
304                }
305            }
306        }
307
308        if !frames.is_empty() {
309            self.pending_ipis.extend(&self.arch.invalidate(&frames));
310        }
311        drop(ManuallyDrop::into_inner(frames));
312        num_unmapped
313    }
314
315    /// Waits for this cursor's previous EPT invalidations to complete on all CPUs.
316    ///
317    /// This synchronizes invalidations issued by [`Self::unmap`] and
318    /// [`Self::protect_next`]. Dropping the cursor does not wait for completion;
319    /// removed frames are retained until invalidation completes regardless.
320    ///
321    /// # Panics
322    ///
323    /// Panics if local IRQs are disabled.
324    pub fn sync_tlb_flush(&mut self) {
325        self.pending_ipis.wait();
326        self.pending_ipis = PendingIpis::new_empty();
327    }
328}
329
330/// A page table mapping guest physical addresses to host physical frames.
331#[derive(Clone, Debug)]
332pub(crate) struct GuestPtConfig {}
333
334// SAFETY: `item_raw_info`, `item_into_raw`, `item_from_raw`, and
335// `item_ref_from_raw` are correctly implemented with respect to the `Item` and
336// `ItemRef` types.
337unsafe impl PageTableConfig for GuestPtConfig {
338    const TOP_LEVEL_INDEX_RANGE: Range<usize> = 0..512;
339
340    type E = PageTableEntry;
341    type C = PagingConsts;
342
343    /// All mappings are tracked untyped frames.
344    type Item = GuestPtItem;
345    type ItemRef<'a> = GuestPtItemRef<'a>;
346
347    fn item_raw_info(item: &Self::Item) -> (Paddr, PagingLevel, PageProperty) {
348        let (frame, prop) = item;
349        (frame.paddr(), frame.map_level(), *prop)
350    }
351
352    unsafe fn item_from_raw(paddr: Paddr, level: PagingLevel, prop: PageProperty) -> Self::Item {
353        debug_assert_eq!(level, 1);
354        // SAFETY: The caller ensures safety.
355        let frame = unsafe { Frame::<dyn AnyUFrameMeta>::from_raw(paddr) };
356        (frame, prop)
357    }
358
359    unsafe fn item_ref_from_raw<'a>(
360        paddr: Paddr,
361        level: PagingLevel,
362        prop: PageProperty,
363    ) -> Self::ItemRef<'a> {
364        debug_assert_eq!(level, 1);
365        // SAFETY: The caller ensures that the frame outlives `'a` and that
366        // the type matches the frame.
367        let frame = unsafe { FrameRef::<dyn AnyUFrameMeta>::borrow_paddr(paddr) };
368        (frame, prop)
369    }
370}
371
372pub(crate) type GuestPtItem = (UFrame, PageProperty);
373pub(crate) type GuestPtItemRef<'a> = (FrameRef<'a, dyn AnyUFrameMeta>, PageProperty);
374
375#[cfg(ktest)]
376mod test {
377    use super::*;
378    use crate::{
379        Error,
380        mm::{CachePolicy, FrameAllocOptions, PAGE_SIZE},
381        task::disable_preempt,
382    };
383
384    #[ktest]
385    fn guest_mapping_lifecycle() {
386        let space = match GuestPhysMemSpace::new() {
387            Ok(space) => space,
388            Err(Error::NotEnoughResources | Error::AccessDenied) => {
389                crate::early_print!(" [skipped: VMX/EPT invalidation unavailable]");
390                return;
391            }
392            Err(err) => panic!("failed to create guest physical memory: {:?}", err),
393        };
394
395        let first = FrameAllocOptions::new().alloc_frame().unwrap();
396        let second = FrameAllocOptions::new().alloc_frame().unwrap();
397        let first_paddr = first.paddr();
398        let second_paddr = second.paddr();
399        let prop = PageProperty::new_guest(PageFlags::RWX, CachePolicy::Writeback);
400        let range = PAGE_SIZE..4 * PAGE_SIZE;
401
402        let guard = disable_preempt();
403        let mut cursor = space.cursor_mut(&guard, &range).unwrap();
404
405        // Test `map`.
406        cursor.map(first.into(), prop);
407        cursor.map(second.into(), prop);
408        assert_eq!(cursor.gpa(), 3 * PAGE_SIZE);
409
410        // Test `query`.
411        cursor.jump(2 * PAGE_SIZE).unwrap();
412        let (queried_range, item) = cursor.query().unwrap();
413        assert_eq!(queried_range, 2 * PAGE_SIZE..3 * PAGE_SIZE);
414        assert_eq!(
415            item.map(|(frame, prop)| (frame.paddr(), prop)),
416            Some((second_paddr, prop))
417        );
418        cursor.jump(PAGE_SIZE).unwrap();
419        let (queried_range, item) = cursor.query().unwrap();
420        assert_eq!(queried_range, PAGE_SIZE..2 * PAGE_SIZE);
421        assert_eq!(
422            item.map(|(frame, prop)| (frame.paddr(), prop)),
423            Some((first_paddr, prop))
424        );
425
426        // Test `protect`.
427        assert_eq!(
428            cursor.protect_next(PAGE_SIZE, &mut |flags| *flags = PageFlags::RX),
429            Some(PAGE_SIZE..2 * PAGE_SIZE)
430        );
431        cursor.jump(PAGE_SIZE).unwrap();
432        assert_eq!(cursor.query().unwrap().1.unwrap().1.flags, PageFlags::RX);
433
434        // Test `unmap`.
435        assert_eq!(cursor.unmap(2 * PAGE_SIZE), 2);
436        cursor.jump(PAGE_SIZE).unwrap();
437        assert!(cursor.find_next(3 * PAGE_SIZE).is_none());
438    }
439}