x86_64/structures/paging/mapper/
recursive_page_table.rs

1//! Access the page tables through a recursively mapped level 4 table.
2
3use core::fmt;
4
5use super::*;
6use crate::registers::control::Cr3;
7use crate::structures::paging::page_table::PageTableLevel;
8use crate::structures::paging::{
9    page::{AddressNotAligned, NotGiantPageSize},
10    page_table::{FrameError, PageTable, PageTableEntry},
11    PageTableIndex,
12};
13
14/// A recursive page table is a last level page table with an entry mapped to the table itself.
15///
16/// This recursive mapping allows accessing all page tables in the hierarchy:
17///
18/// - To access the level 4 page table, we “loop“ (i.e. follow the recursively mapped entry) four
19///   times.
20/// - To access a level 3 page table, we “loop” three times and then use the level 4 index.
21/// - To access a level 2 page table, we “loop” two times, then use the level 4 index, then the
22///   level 3 index.
23/// - To access a level 1 page table, we “loop” once, then use the level 4 index, then the
24///   level 3 index, then the level 2 index.
25///
26/// This struct implements the `Mapper` trait.
27///
28/// The page table flags `PRESENT` and `WRITABLE` are always set for higher level page table
29/// entries, even if not specified, because the design of the recursive page table requires it.
30#[derive(Debug)]
31pub struct RecursivePageTable<'a> {
32    p4: &'a mut PageTable,
33    recursive_index: PageTableIndex,
34}
35
36impl<'a> RecursivePageTable<'a> {
37    /// Creates a new RecursivePageTable from the passed level 4 PageTable.
38    ///
39    /// The page table must be recursively mapped, that means:
40    ///
41    /// - The page table must have one recursive entry, i.e. an entry that points to the table
42    ///   itself.
43    ///     - The reference must use that “loop”, i.e. be of the form `0o_xxx_xxx_xxx_xxx_0000`
44    ///       where `xxx` is the recursive entry.
45    /// - The page table must be active, i.e. the CR3 register must contain its physical address.
46    ///
47    /// Otherwise `Err(())` is returned.
48    ///
49    /// ## Safety
50    ///
51    /// Note that creating a `PageTable` with recursive index 511 is unsound
52    /// because allocating the last byte of the address space can lead to pointer
53    /// overflows and undefined behavior. For more details, see the discussions
54    /// [on Zulip](https://rust-lang.zulipchat.com/#narrow/stream/136281-t-opsem/topic/end-of-address-space)
55    /// and [in the `unsafe-code-guidelines ` repo](https://github.com/rust-lang/unsafe-code-guidelines/issues/420).
56    #[inline]
57    pub fn new(table: &'a mut PageTable) -> Result<Self, InvalidPageTable> {
58        let page = Page::containing_address(VirtAddr::new(table as *const _ as u64));
59        let recursive_index = page.p4_index();
60
61        if page.p3_index() != recursive_index
62            || page.p2_index() != recursive_index
63            || page.p1_index() != recursive_index
64        {
65            return Err(InvalidPageTable::NotRecursive);
66        }
67        if Ok(Cr3::read().0) != table[recursive_index].frame() {
68            return Err(InvalidPageTable::NotActive);
69        }
70
71        Ok(RecursivePageTable {
72            p4: table,
73            recursive_index,
74        })
75    }
76
77    /// Creates a new RecursivePageTable without performing any checks.
78    ///
79    /// ## Safety
80    ///
81    /// The given page table must be a level 4 page table that is active in the
82    /// CPU (i.e. loaded in the CR3 register). The `recursive_index` parameter
83    /// must be the index of the recursively mapped entry of that page table.
84    #[inline]
85    pub unsafe fn new_unchecked(table: &'a mut PageTable, recursive_index: PageTableIndex) -> Self {
86        RecursivePageTable {
87            p4: table,
88            recursive_index,
89        }
90    }
91
92    /// Returns an immutable reference to the wrapped level 4 `PageTable` instance.
93    pub fn level_4_table(&self) -> &PageTable {
94        self.p4
95    }
96
97    /// Returns a mutable reference to the wrapped level 4 `PageTable` instance.
98    pub fn level_4_table_mut(&mut self) -> &mut PageTable {
99        self.p4
100    }
101
102    /// Internal helper function to create the page table of the next level if needed.
103    ///
104    /// If the passed entry is unused, a new frame is allocated from the given allocator, zeroed,
105    /// and the entry is updated to that address. If the passed entry is already mapped, the next
106    /// table is returned directly.
107    ///
108    /// The page table flags `PRESENT` and `WRITABLE` are always set for higher level page table
109    /// entries, even if not specified in the `insert_flags`, because the design of the
110    /// recursive page table requires it.
111    ///
112    /// The `next_page_table` page must be the page of the next page table in the hierarchy.
113    ///
114    /// Returns `MapToError::FrameAllocationFailed` if the entry is unused and the allocator
115    /// returned `None`. Returns `MapToError::ParentEntryHugePage` if the `HUGE_PAGE` flag is set
116    /// in the passed entry.
117    unsafe fn create_next_table<'b, A, S: PageSize>(
118        entry: &'b mut PageTableEntry,
119        next_table_page: Page,
120        insert_flags: PageTableFlags,
121        allocator: &mut A,
122    ) -> Result<&'b mut PageTable, MapToError<S>>
123    where
124        A: FrameAllocator<Size4KiB> + ?Sized,
125    {
126        /// This inner function is used to limit the scope of `unsafe`.
127        ///
128        /// This is a safe function, so we need to use `unsafe` blocks when we do something unsafe.
129        fn inner<'b, A, S: PageSize>(
130            entry: &'b mut PageTableEntry,
131            next_table_page: Page,
132            insert_flags: PageTableFlags,
133            allocator: &mut A,
134        ) -> Result<&'b mut PageTable, MapToError<S>>
135        where
136            A: FrameAllocator<Size4KiB> + ?Sized,
137        {
138            use crate::structures::paging::PageTableFlags as Flags;
139
140            let created;
141
142            if entry.is_unused() {
143                if let Some(frame) = allocator.allocate_frame() {
144                    entry.set_frame(frame, Flags::PRESENT | Flags::WRITABLE | insert_flags);
145                    created = true;
146                } else {
147                    return Err(MapToError::FrameAllocationFailed);
148                }
149            } else {
150                if !insert_flags.is_empty() && !entry.flags().contains(insert_flags) {
151                    entry.set_flags(entry.flags() | insert_flags);
152                }
153                created = false;
154            }
155            if entry.flags().contains(Flags::HUGE_PAGE) {
156                return Err(MapToError::ParentEntryHugePage);
157            }
158
159            let page_table_ptr = next_table_page.start_address().as_mut_ptr();
160            let page_table: &mut PageTable = unsafe { &mut *(page_table_ptr) };
161            if created {
162                page_table.zero();
163            }
164            Ok(page_table)
165        }
166
167        inner(entry, next_table_page, insert_flags, allocator)
168    }
169}
170
171impl Mapper<Size1GiB> for RecursivePageTable<'_> {
172    #[inline]
173    unsafe fn map_to_with_table_flags<A>(
174        &mut self,
175        page: Page<Size1GiB>,
176        frame: PhysFrame<Size1GiB>,
177        flags: PageTableFlags,
178        parent_table_flags: PageTableFlags,
179        allocator: &mut A,
180    ) -> Result<MapperFlush<Size1GiB>, MapToError<Size1GiB>>
181    where
182        A: FrameAllocator<Size4KiB> + ?Sized,
183    {
184        use crate::structures::paging::PageTableFlags as Flags;
185        let p4 = &mut self.p4;
186
187        let p3_page = p3_page(page, self.recursive_index);
188        let p3 = unsafe {
189            Self::create_next_table(
190                &mut p4[page.p4_index()],
191                p3_page,
192                parent_table_flags,
193                allocator,
194            )?
195        };
196
197        if !p3[page.p3_index()].is_unused() {
198            return Err(MapToError::PageAlreadyMapped(frame));
199        }
200        p3[page.p3_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE);
201
202        Ok(MapperFlush::new(page))
203    }
204
205    fn unmap(
206        &mut self,
207        page: Page<Size1GiB>,
208    ) -> Result<(PhysFrame<Size1GiB>, MapperFlush<Size1GiB>), UnmapError> {
209        let p4 = &mut self.p4;
210        let p4_entry = &p4[page.p4_index()];
211
212        p4_entry.frame().map_err(|err| match err {
213            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
214            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
215        })?;
216
217        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
218        let p3_entry = &mut p3[page.p3_index()];
219        let flags = p3_entry.flags();
220
221        if !flags.contains(PageTableFlags::PRESENT) {
222            return Err(UnmapError::PageNotMapped);
223        }
224        if !flags.contains(PageTableFlags::HUGE_PAGE) {
225            return Err(UnmapError::ParentEntryHugePage);
226        }
227
228        let frame = PhysFrame::from_start_address(p3_entry.addr())
229            .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(p3_entry.addr()))?;
230
231        p3_entry.set_unused();
232        Ok((frame, MapperFlush::new(page)))
233    }
234
235    unsafe fn update_flags(
236        &mut self,
237        page: Page<Size1GiB>,
238        flags: PageTableFlags,
239    ) -> Result<MapperFlush<Size1GiB>, FlagUpdateError> {
240        use crate::structures::paging::PageTableFlags as Flags;
241        let p4 = &mut self.p4;
242
243        if p4[page.p4_index()].is_unused() {
244            return Err(FlagUpdateError::PageNotMapped);
245        }
246
247        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
248
249        if p3[page.p3_index()].is_unused() {
250            return Err(FlagUpdateError::PageNotMapped);
251        }
252        p3[page.p3_index()].set_flags(flags | Flags::HUGE_PAGE);
253
254        Ok(MapperFlush::new(page))
255    }
256
257    unsafe fn set_flags_p4_entry(
258        &mut self,
259        page: Page<Size1GiB>,
260        flags: PageTableFlags,
261    ) -> Result<MapperFlushAll, FlagUpdateError> {
262        let p4 = &mut self.p4;
263        let p4_entry = &mut p4[page.p4_index()];
264
265        if p4_entry.is_unused() {
266            return Err(FlagUpdateError::PageNotMapped);
267        }
268
269        p4_entry.set_flags(flags);
270
271        Ok(MapperFlushAll::new())
272    }
273
274    unsafe fn set_flags_p3_entry(
275        &mut self,
276        _page: Page<Size1GiB>,
277        _flags: PageTableFlags,
278    ) -> Result<MapperFlushAll, FlagUpdateError> {
279        Err(FlagUpdateError::ParentEntryHugePage)
280    }
281
282    unsafe fn set_flags_p2_entry(
283        &mut self,
284        _page: Page<Size1GiB>,
285        _flags: PageTableFlags,
286    ) -> Result<MapperFlushAll, FlagUpdateError> {
287        Err(FlagUpdateError::ParentEntryHugePage)
288    }
289
290    fn translate_page(&self, page: Page<Size1GiB>) -> Result<PhysFrame<Size1GiB>, TranslateError> {
291        let p4 = &self.p4;
292
293        if p4[page.p4_index()].is_unused() {
294            return Err(TranslateError::PageNotMapped);
295        }
296
297        let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
298        let p3_entry = &p3[page.p3_index()];
299
300        if p3_entry.is_unused() {
301            return Err(TranslateError::PageNotMapped);
302        }
303
304        PhysFrame::from_start_address(p3_entry.addr())
305            .map_err(|AddressNotAligned| TranslateError::InvalidFrameAddress(p3_entry.addr()))
306    }
307}
308
309impl Mapper<Size2MiB> for RecursivePageTable<'_> {
310    #[inline]
311    unsafe fn map_to_with_table_flags<A>(
312        &mut self,
313        page: Page<Size2MiB>,
314        frame: PhysFrame<Size2MiB>,
315        flags: PageTableFlags,
316        parent_table_flags: PageTableFlags,
317        allocator: &mut A,
318    ) -> Result<MapperFlush<Size2MiB>, MapToError<Size2MiB>>
319    where
320        A: FrameAllocator<Size4KiB> + ?Sized,
321    {
322        use crate::structures::paging::PageTableFlags as Flags;
323        let p4 = &mut self.p4;
324
325        let p3_page = p3_page(page, self.recursive_index);
326        let p3 = unsafe {
327            Self::create_next_table(
328                &mut p4[page.p4_index()],
329                p3_page,
330                parent_table_flags,
331                allocator,
332            )?
333        };
334
335        let p2_page = p2_page(page, self.recursive_index);
336        let p2 = unsafe {
337            Self::create_next_table(
338                &mut p3[page.p3_index()],
339                p2_page,
340                parent_table_flags,
341                allocator,
342            )?
343        };
344
345        if !p2[page.p2_index()].is_unused() {
346            return Err(MapToError::PageAlreadyMapped(frame));
347        }
348        p2[page.p2_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE);
349
350        Ok(MapperFlush::new(page))
351    }
352
353    fn unmap(
354        &mut self,
355        page: Page<Size2MiB>,
356    ) -> Result<(PhysFrame<Size2MiB>, MapperFlush<Size2MiB>), UnmapError> {
357        let p4 = &mut self.p4;
358        let p4_entry = &p4[page.p4_index()];
359        p4_entry.frame().map_err(|err| match err {
360            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
361            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
362        })?;
363
364        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
365        let p3_entry = &p3[page.p3_index()];
366        p3_entry.frame().map_err(|err| match err {
367            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
368            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
369        })?;
370
371        let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
372        let p2_entry = &mut p2[page.p2_index()];
373        let flags = p2_entry.flags();
374
375        if !flags.contains(PageTableFlags::PRESENT) {
376            return Err(UnmapError::PageNotMapped);
377        }
378        if !flags.contains(PageTableFlags::HUGE_PAGE) {
379            return Err(UnmapError::ParentEntryHugePage);
380        }
381
382        let frame = PhysFrame::from_start_address(p2_entry.addr())
383            .map_err(|AddressNotAligned| UnmapError::InvalidFrameAddress(p2_entry.addr()))?;
384
385        p2_entry.set_unused();
386        Ok((frame, MapperFlush::new(page)))
387    }
388
389    unsafe fn update_flags(
390        &mut self,
391        page: Page<Size2MiB>,
392        flags: PageTableFlags,
393    ) -> Result<MapperFlush<Size2MiB>, FlagUpdateError> {
394        use crate::structures::paging::PageTableFlags as Flags;
395        let p4 = &mut self.p4;
396
397        if p4[page.p4_index()].is_unused() {
398            return Err(FlagUpdateError::PageNotMapped);
399        }
400
401        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
402
403        if p3[page.p3_index()].is_unused() {
404            return Err(FlagUpdateError::PageNotMapped);
405        }
406
407        let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
408
409        if p2[page.p2_index()].is_unused() {
410            return Err(FlagUpdateError::PageNotMapped);
411        }
412
413        p2[page.p2_index()].set_flags(flags | Flags::HUGE_PAGE);
414
415        Ok(MapperFlush::new(page))
416    }
417
418    unsafe fn set_flags_p4_entry(
419        &mut self,
420        page: Page<Size2MiB>,
421        flags: PageTableFlags,
422    ) -> Result<MapperFlushAll, FlagUpdateError> {
423        let p4 = &mut self.p4;
424        let p4_entry = &mut p4[page.p4_index()];
425
426        if p4_entry.is_unused() {
427            return Err(FlagUpdateError::PageNotMapped);
428        }
429
430        p4_entry.set_flags(flags);
431
432        Ok(MapperFlushAll::new())
433    }
434
435    unsafe fn set_flags_p3_entry(
436        &mut self,
437        page: Page<Size2MiB>,
438        flags: PageTableFlags,
439    ) -> Result<MapperFlushAll, FlagUpdateError> {
440        let p4 = &mut self.p4;
441
442        if p4[page.p4_index()].is_unused() {
443            return Err(FlagUpdateError::PageNotMapped);
444        }
445
446        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
447        let p3_entry = &mut p3[page.p3_index()];
448
449        if p3_entry.is_unused() {
450            return Err(FlagUpdateError::PageNotMapped);
451        }
452
453        p3_entry.set_flags(flags);
454
455        Ok(MapperFlushAll::new())
456    }
457
458    unsafe fn set_flags_p2_entry(
459        &mut self,
460        _page: Page<Size2MiB>,
461        _flags: PageTableFlags,
462    ) -> Result<MapperFlushAll, FlagUpdateError> {
463        Err(FlagUpdateError::ParentEntryHugePage)
464    }
465
466    fn translate_page(&self, page: Page<Size2MiB>) -> Result<PhysFrame<Size2MiB>, TranslateError> {
467        let p4 = &self.p4;
468
469        if p4[page.p4_index()].is_unused() {
470            return Err(TranslateError::PageNotMapped);
471        }
472
473        let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
474        let p3_entry = &p3[page.p3_index()];
475
476        if p3_entry.is_unused() {
477            return Err(TranslateError::PageNotMapped);
478        }
479
480        let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) };
481        let p2_entry = &p2[page.p2_index()];
482
483        if p2_entry.is_unused() {
484            return Err(TranslateError::PageNotMapped);
485        }
486
487        PhysFrame::from_start_address(p2_entry.addr())
488            .map_err(|AddressNotAligned| TranslateError::InvalidFrameAddress(p2_entry.addr()))
489    }
490}
491
492impl Mapper<Size4KiB> for RecursivePageTable<'_> {
493    #[inline]
494    unsafe fn map_to_with_table_flags<A>(
495        &mut self,
496        page: Page<Size4KiB>,
497        frame: PhysFrame<Size4KiB>,
498        flags: PageTableFlags,
499        parent_table_flags: PageTableFlags,
500        allocator: &mut A,
501    ) -> Result<MapperFlush<Size4KiB>, MapToError<Size4KiB>>
502    where
503        A: FrameAllocator<Size4KiB> + ?Sized,
504    {
505        let p4 = &mut self.p4;
506
507        let p3_page = p3_page(page, self.recursive_index);
508        let p3 = unsafe {
509            Self::create_next_table(
510                &mut p4[page.p4_index()],
511                p3_page,
512                parent_table_flags,
513                allocator,
514            )?
515        };
516
517        let p2_page = p2_page(page, self.recursive_index);
518        let p2 = unsafe {
519            Self::create_next_table(
520                &mut p3[page.p3_index()],
521                p2_page,
522                parent_table_flags,
523                allocator,
524            )?
525        };
526
527        let p1_page = p1_page(page, self.recursive_index);
528        let p1 = unsafe {
529            Self::create_next_table(
530                &mut p2[page.p2_index()],
531                p1_page,
532                parent_table_flags,
533                allocator,
534            )?
535        };
536
537        if !p1[page.p1_index()].is_unused() {
538            return Err(MapToError::PageAlreadyMapped(frame));
539        }
540        p1[page.p1_index()].set_frame(frame, flags);
541
542        Ok(MapperFlush::new(page))
543    }
544
545    fn unmap(
546        &mut self,
547        page: Page<Size4KiB>,
548    ) -> Result<(PhysFrame<Size4KiB>, MapperFlush<Size4KiB>), UnmapError> {
549        let p4 = &mut self.p4;
550        let p4_entry = &p4[page.p4_index()];
551        p4_entry.frame().map_err(|err| match err {
552            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
553            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
554        })?;
555
556        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
557        let p3_entry = &p3[page.p3_index()];
558        p3_entry.frame().map_err(|err| match err {
559            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
560            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
561        })?;
562
563        let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
564        let p2_entry = &p2[page.p2_index()];
565        p2_entry.frame().map_err(|err| match err {
566            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
567            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
568        })?;
569
570        let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) };
571        let p1_entry = &mut p1[page.p1_index()];
572
573        let frame = p1_entry.frame().map_err(|err| match err {
574            FrameError::FrameNotPresent => UnmapError::PageNotMapped,
575            FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
576        })?;
577
578        p1_entry.set_unused();
579        Ok((frame, MapperFlush::new(page)))
580    }
581
582    unsafe fn update_flags(
583        &mut self,
584        page: Page<Size4KiB>,
585        flags: PageTableFlags,
586    ) -> Result<MapperFlush<Size4KiB>, FlagUpdateError> {
587        let p4 = &mut self.p4;
588
589        if p4[page.p4_index()].is_unused() {
590            return Err(FlagUpdateError::PageNotMapped);
591        }
592
593        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
594
595        if p3[page.p3_index()].is_unused() {
596            return Err(FlagUpdateError::PageNotMapped);
597        }
598
599        let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
600
601        if p2[page.p2_index()].is_unused() {
602            return Err(FlagUpdateError::PageNotMapped);
603        }
604
605        let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) };
606
607        if p1[page.p1_index()].is_unused() {
608            return Err(FlagUpdateError::PageNotMapped);
609        }
610
611        p1[page.p1_index()].set_flags(flags);
612
613        Ok(MapperFlush::new(page))
614    }
615
616    unsafe fn set_flags_p4_entry(
617        &mut self,
618        page: Page<Size4KiB>,
619        flags: PageTableFlags,
620    ) -> Result<MapperFlushAll, FlagUpdateError> {
621        let p4 = &mut self.p4;
622        let p4_entry = &mut p4[page.p4_index()];
623
624        if p4_entry.is_unused() {
625            return Err(FlagUpdateError::PageNotMapped);
626        }
627
628        p4_entry.set_flags(flags);
629
630        Ok(MapperFlushAll::new())
631    }
632
633    unsafe fn set_flags_p3_entry(
634        &mut self,
635        page: Page<Size4KiB>,
636        flags: PageTableFlags,
637    ) -> Result<MapperFlushAll, FlagUpdateError> {
638        let p4 = &mut self.p4;
639
640        if p4[page.p4_index()].is_unused() {
641            return Err(FlagUpdateError::PageNotMapped);
642        }
643
644        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
645        let p3_entry = &mut p3[page.p3_index()];
646
647        if p3_entry.is_unused() {
648            return Err(FlagUpdateError::PageNotMapped);
649        }
650
651        p3_entry.set_flags(flags);
652
653        Ok(MapperFlushAll::new())
654    }
655
656    unsafe fn set_flags_p2_entry(
657        &mut self,
658        page: Page<Size4KiB>,
659        flags: PageTableFlags,
660    ) -> Result<MapperFlushAll, FlagUpdateError> {
661        let p4 = &mut self.p4;
662
663        if p4[page.p4_index()].is_unused() {
664            return Err(FlagUpdateError::PageNotMapped);
665        }
666
667        let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
668
669        if p3[page.p3_index()].is_unused() {
670            return Err(FlagUpdateError::PageNotMapped);
671        }
672
673        let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
674        let p2_entry = &mut p2[page.p2_index()];
675
676        if p2_entry.is_unused() {
677            return Err(FlagUpdateError::PageNotMapped);
678        }
679
680        p2_entry.set_flags(flags);
681
682        Ok(MapperFlushAll::new())
683    }
684
685    fn translate_page(&self, page: Page<Size4KiB>) -> Result<PhysFrame<Size4KiB>, TranslateError> {
686        let p4 = &self.p4;
687
688        if p4[page.p4_index()].is_unused() {
689            return Err(TranslateError::PageNotMapped);
690        }
691
692        let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
693        let p3_entry = &p3[page.p3_index()];
694
695        if p3_entry.is_unused() {
696            return Err(TranslateError::PageNotMapped);
697        }
698
699        let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) };
700        let p2_entry = &p2[page.p2_index()];
701
702        if p2_entry.is_unused() {
703            return Err(TranslateError::PageNotMapped);
704        }
705
706        let p1 = unsafe { &*(p1_ptr(page, self.recursive_index)) };
707        let p1_entry = &p1[page.p1_index()];
708
709        if p1_entry.is_unused() {
710            return Err(TranslateError::PageNotMapped);
711        }
712
713        PhysFrame::from_start_address(p1_entry.addr())
714            .map_err(|AddressNotAligned| TranslateError::InvalidFrameAddress(p1_entry.addr()))
715    }
716}
717
718impl Translate for RecursivePageTable<'_> {
719    #[allow(clippy::inconsistent_digit_grouping)]
720    fn translate(&self, addr: VirtAddr) -> TranslateResult {
721        let page = Page::containing_address(addr);
722
723        let p4 = &self.p4;
724        let p4_entry = &p4[addr.p4_index()];
725        if p4_entry.is_unused() {
726            return TranslateResult::NotMapped;
727        }
728        if p4_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
729            panic!("level 4 entry has huge page bit set")
730        }
731
732        let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
733        let p3_entry = &p3[addr.p3_index()];
734        if p3_entry.is_unused() {
735            return TranslateResult::NotMapped;
736        }
737        if p3_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
738            let entry = &p3[addr.p3_index()];
739            let frame = PhysFrame::containing_address(entry.addr());
740            #[allow(clippy::unusual_byte_groupings)]
741            let offset = addr.as_u64() & 0o_777_777_7777;
742            let flags = entry.flags();
743            return TranslateResult::Mapped {
744                frame: MappedFrame::Size1GiB(frame),
745                offset,
746                flags,
747            };
748        }
749
750        let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) };
751        let p2_entry = &p2[addr.p2_index()];
752        if p2_entry.is_unused() {
753            return TranslateResult::NotMapped;
754        }
755        if p2_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
756            let entry = &p2[addr.p2_index()];
757            let frame = PhysFrame::containing_address(entry.addr());
758            #[allow(clippy::unusual_byte_groupings)]
759            let offset = addr.as_u64() & 0o_777_7777;
760            let flags = entry.flags();
761            return TranslateResult::Mapped {
762                frame: MappedFrame::Size2MiB(frame),
763                offset,
764                flags,
765            };
766        }
767
768        let p1 = unsafe { &*(p1_ptr(page, self.recursive_index)) };
769        let p1_entry = &p1[addr.p1_index()];
770        if p1_entry.is_unused() {
771            return TranslateResult::NotMapped;
772        }
773        if p1_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
774            panic!("level 1 entry has huge page bit set")
775        }
776
777        let frame = match PhysFrame::from_start_address(p1_entry.addr()) {
778            Ok(frame) => frame,
779            Err(AddressNotAligned) => return TranslateResult::InvalidFrameAddress(p1_entry.addr()),
780        };
781        let offset = u64::from(addr.page_offset());
782        let flags = p1_entry.flags();
783        TranslateResult::Mapped {
784            frame: MappedFrame::Size4KiB(frame),
785            offset,
786            flags,
787        }
788    }
789}
790
791impl CleanUp for RecursivePageTable<'_> {
792    #[inline]
793    unsafe fn clean_up<D>(&mut self, frame_deallocator: &mut D)
794    where
795        D: FrameDeallocator<Size4KiB>,
796    {
797        unsafe {
798            self.clean_up_addr_range(
799                PageRangeInclusive {
800                    start: Page::from_start_address(VirtAddr::new(0)).unwrap(),
801                    end: Page::from_start_address(VirtAddr::new(0xffff_ffff_ffff_f000)).unwrap(),
802                },
803                frame_deallocator,
804            )
805        }
806    }
807
808    unsafe fn clean_up_addr_range<D>(
809        &mut self,
810        range: PageRangeInclusive,
811        frame_deallocator: &mut D,
812    ) where
813        D: FrameDeallocator<Size4KiB>,
814    {
815        fn clean_up(
816            recursive_index: PageTableIndex,
817            page_table: &mut PageTable,
818            level: PageTableLevel,
819            range: PageRangeInclusive,
820            frame_deallocator: &mut impl FrameDeallocator<Size4KiB>,
821        ) -> bool {
822            if range.is_empty() {
823                return false;
824            }
825
826            let table_addr = range
827                .start
828                .start_address()
829                .align_down(level.table_address_space_alignment());
830
831            let start = range.start.page_table_index(level);
832            let end = range.end.page_table_index(level);
833
834            if let Some(next_level) = level.next_lower_level() {
835                let offset_per_entry = level.entry_address_space_alignment();
836                for (i, entry) in page_table
837                    .iter_mut()
838                    .enumerate()
839                    .take(usize::from(end) + 1)
840                    .skip(usize::from(start))
841                    .filter(|(i, _)| {
842                        !(level == PageTableLevel::Four && *i == recursive_index.into())
843                    })
844                {
845                    if let Ok(frame) = entry.frame() {
846                        let start = VirtAddr::forward_checked_impl(
847                            table_addr,
848                            (offset_per_entry as usize) * i,
849                        )
850                        .unwrap();
851                        let end = start + (offset_per_entry - 1);
852                        let start = Page::<Size4KiB>::containing_address(start);
853                        let start = start.max(range.start);
854                        let end = Page::<Size4KiB>::containing_address(end);
855                        let end = end.min(range.end);
856                        let page_table =
857                            [p1_ptr, p2_ptr, p3_ptr][level as usize - 2](start, recursive_index);
858                        let page_table = unsafe { &mut *page_table };
859                        if clean_up(
860                            recursive_index,
861                            page_table,
862                            next_level,
863                            Page::range_inclusive(start, end),
864                            frame_deallocator,
865                        ) {
866                            entry.set_unused();
867                            unsafe {
868                                frame_deallocator.deallocate_frame(frame);
869                            }
870                        }
871                    }
872                }
873            }
874
875            page_table.iter().all(PageTableEntry::is_unused)
876        }
877
878        clean_up(
879            self.recursive_index,
880            self.level_4_table_mut(),
881            PageTableLevel::Four,
882            range,
883            frame_deallocator,
884        );
885    }
886}
887
888/// The given page table was not suitable to create a `RecursivePageTable`.
889#[derive(Debug)]
890pub enum InvalidPageTable {
891    /// The given page table was not at an recursive address.
892    ///
893    /// The page table address must be of the form `0o_xxx_xxx_xxx_xxx_0000` where `xxx`
894    /// is the recursive entry.
895    NotRecursive,
896    /// The given page table was not active on the CPU.
897    ///
898    /// The recursive page table design requires that the given level 4 table is active
899    /// on the CPU because otherwise it's not possible to access the other page tables
900    /// through recursive memory addresses.
901    NotActive,
902}
903
904impl fmt::Display for InvalidPageTable {
905    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
906        match self {
907            InvalidPageTable::NotRecursive => {
908                write!(f, "given page table address is not recursive")
909            }
910            InvalidPageTable::NotActive => write!(f, "given page table is not active on the CPU"),
911        }
912    }
913}
914
915#[inline]
916fn p3_ptr<S: PageSize>(page: Page<S>, recursive_index: PageTableIndex) -> *mut PageTable {
917    p3_page(page, recursive_index).start_address().as_mut_ptr()
918}
919
920#[inline]
921fn p3_page<S: PageSize>(page: Page<S>, recursive_index: PageTableIndex) -> Page {
922    Page::from_page_table_indices(
923        recursive_index,
924        recursive_index,
925        recursive_index,
926        page.p4_index(),
927    )
928}
929
930#[inline]
931fn p2_ptr<S: NotGiantPageSize>(page: Page<S>, recursive_index: PageTableIndex) -> *mut PageTable {
932    p2_page(page, recursive_index).start_address().as_mut_ptr()
933}
934
935#[inline]
936fn p2_page<S: NotGiantPageSize>(page: Page<S>, recursive_index: PageTableIndex) -> Page {
937    Page::from_page_table_indices(
938        recursive_index,
939        recursive_index,
940        page.p4_index(),
941        page.p3_index(),
942    )
943}
944
945#[inline]
946fn p1_ptr(page: Page<Size4KiB>, recursive_index: PageTableIndex) -> *mut PageTable {
947    p1_page(page, recursive_index).start_address().as_mut_ptr()
948}
949
950#[inline]
951fn p1_page(page: Page<Size4KiB>, recursive_index: PageTableIndex) -> Page {
952    Page::from_page_table_indices(
953        recursive_index,
954        page.p4_index(),
955        page.p3_index(),
956        page.p2_index(),
957    )
958}
x86_64/structures/paging/mapper/recursive_page_table.rs

x86_64/structures/paging/mapper/
recursive_page_table.rs
