Skip to main content

ostd/mm/kspace/
mod.rs

1// SPDX-License-Identifier: MPL-2.0
2
3//! Kernel memory space management.
4//!
5//! The kernel memory space is currently managed as follows, if the
6//! address width is 48 bits (with 47 bits kernel space).
7//!
8//! TODO: the cap of linear mapping (the start of vm alloc) are raised
9//! to workaround for high IO in TDX. We need actual vm alloc API to have
10//! a proper fix.
11//!
12//! ```text
13//! +-+ <- the highest used address (0xffff_ffff_ffff_0000)
14//! | |         For the kernel code, 1 GiB.
15//! +-+ <- 0xffff_ffff_8000_0000
16//! | |
17//! | |         Unused hole.
18//! +-+ <- 0xffff_e100_0000_0000
19//! | |         For frame metadata, 1 TiB.
20//! +-+ <- 0xffff_e000_0000_0000
21//! | |         For [`KVirtArea`], 32 TiB.
22//! +-+ <- the middle of the higher half (0xffff_c000_0000_0000)
23//! | |
24//! | |
25//! | |
26//! | |         For linear mappings, 64 TiB.
27//! | |         Mapped physical addresses are untracked.
28//! | |
29//! | |
30//! | |
31//! +-+ <- the base of high canonical address (0xffff_8000_0000_0000)
32//! ```
33//!
34//! If the address width is (according to [`crate::arch::mm::PagingConsts`])
35//! 39 bits or 57 bits, the memory space just adjust proportionally.
36
37#![cfg_attr(target_arch = "loongarch64", expect(unused_imports))]
38
39pub(crate) mod kvirt_area;
40
41use core::ops::Range;
42
43use spin::Once;
44
45#[cfg(ktest)]
46mod test;
47
48use super::{
49    Frame, HasSize, Paddr, PagingConstsTrait, Vaddr,
50    frame::{
51        Segment,
52        meta::{AnyFrameMeta, MetaPageMeta, mapping},
53    },
54    page_prop::{CachePolicy, PageFlags, PageProperty, PrivilegedPageFlags},
55    page_table::{PageTable, PageTableConfig, largest_pages, max_page_level},
56};
57use crate::{
58    arch::mm::{PageTableEntry, PagingConsts},
59    boot::memory_region::MemoryRegionType,
60    const_assert, info,
61    mm::{HasPaddr, PAGE_SIZE, PagingLevel, frame::FrameRef},
62    task::disable_preempt,
63};
64
65// The shortest supported address width is 39 bits. So the literal
66// values are written for 39 bits address width and we adjust the values
67// by arithmetic left shift.
68const_assert!(PagingConsts::ADDRESS_WIDTH >= 39);
69const ADDR_WIDTH_SHIFT: usize = PagingConsts::ADDRESS_WIDTH - 39;
70
71/// Start of the kernel address space.
72#[cfg(not(target_arch = "loongarch64"))]
73pub(super) const KERNEL_BASE_VADDR: Vaddr = 0xffff_ffc0_0000_0000 << ADDR_WIDTH_SHIFT;
74#[cfg(target_arch = "loongarch64")]
75pub(super) const KERNEL_BASE_VADDR: Vaddr = 0x9000_0000_0000_0000;
76/// End of the kernel address space (non inclusive).
77pub(super) const KERNEL_END_VADDR: Vaddr = 0xffff_ffff_ffff_0000;
78
79/// The maximum virtual address of user space (non inclusive).
80///
81/// A typical way to reserve half of the address space for the kernel is
82/// to use the highest `ADDRESS_WIDTH`-bit virtual address space.
83///
84/// Also, the top page is not regarded as usable since it's a workaround
85/// for some x86_64 CPUs' bugs. See
86/// <https://github.com/torvalds/linux/blob/480e035fc4c714fb5536e64ab9db04fedc89e910/arch/x86/include/asm/page_64.h#L68-L78>
87/// for the rationale.
88pub const MAX_USERSPACE_VADDR: Vaddr = (0x0000_0040_0000_0000 << ADDR_WIDTH_SHIFT) - PAGE_SIZE;
89
90/// The kernel address space.
91///
92/// They are the high canonical addresses (i.e., the negative part of the
93/// address space, with the most significant bits in the addresses set).
94pub const KERNEL_VADDR_RANGE: Range<Vaddr> = KERNEL_BASE_VADDR..KERNEL_END_VADDR;
95
96/// The offset from physical addresses to kernel image virtual addresses.
97///
98/// On RISC-V, the bootstrap computes this offset from the actual load address.
99/// Other architectures currently use a fixed offset.
100pub(crate) fn kernel_loaded_offset() -> usize {
101    #[cfg(target_arch = "riscv64")]
102    {
103        unsafe extern "C" {
104            static __kernel_loaded_offset: usize;
105        }
106        // SAFETY: The BSP initializes this value before entering Rust, and it
107        // is never modified afterwards. APs are started after initialization.
108        unsafe { __kernel_loaded_offset }
109    }
110    #[cfg(not(target_arch = "riscv64"))]
111    {
112        KERNEL_CODE_BASE_VADDR
113    }
114}
115
116#[cfg(target_arch = "x86_64")]
117const KERNEL_CODE_BASE_VADDR: usize = 0xffff_ffff_8000_0000;
118#[cfg(target_arch = "aarch64")]
119const KERNEL_CODE_BASE_VADDR: usize = 0xffff_ffff_0000_0000;
120#[cfg(target_arch = "loongarch64")]
121const KERNEL_CODE_BASE_VADDR: usize = 0x9000_0000_0000_0000;
122
123const FRAME_METADATA_CAP_VADDR: Vaddr = 0xffff_fff0_8000_0000 << ADDR_WIDTH_SHIFT;
124const FRAME_METADATA_BASE_VADDR: Vaddr = 0xffff_fff0_0000_0000 << ADDR_WIDTH_SHIFT;
125pub(super) const FRAME_METADATA_RANGE: Range<Vaddr> =
126    FRAME_METADATA_BASE_VADDR..FRAME_METADATA_CAP_VADDR;
127
128const VMALLOC_BASE_VADDR: Vaddr = 0xffff_ffe0_0000_0000 << ADDR_WIDTH_SHIFT;
129pub(super) const VMALLOC_VADDR_RANGE: Range<Vaddr> = VMALLOC_BASE_VADDR..FRAME_METADATA_BASE_VADDR;
130
131/// The base address of the linear mapping of all physical
132/// memory in the kernel address space.
133#[cfg(not(target_arch = "loongarch64"))]
134pub(crate) const LINEAR_MAPPING_BASE_VADDR: Vaddr = 0xffff_ffc0_0000_0000 << ADDR_WIDTH_SHIFT;
135#[cfg(target_arch = "loongarch64")]
136pub(crate) const LINEAR_MAPPING_BASE_VADDR: Vaddr = 0x9000_0000_0000_0000;
137pub(crate) const LINEAR_MAPPING_VADDR_RANGE: Range<Vaddr> =
138    LINEAR_MAPPING_BASE_VADDR..VMALLOC_BASE_VADDR;
139
140/// Convert physical address to virtual address using offset, only available inside `ostd`
141pub(crate) fn paddr_to_vaddr(pa: Paddr) -> usize {
142    debug_assert!(pa < VMALLOC_BASE_VADDR - LINEAR_MAPPING_BASE_VADDR);
143    pa + LINEAR_MAPPING_BASE_VADDR
144}
145
146/// The kernel page table instance.
147///
148/// It manages the kernel mapping of all address spaces by sharing the kernel part. And it
149/// is unlikely to be activated.
150pub(super) static KERNEL_PAGE_TABLE: Once<PageTable<KernelPtConfig>> = Once::new();
151
152#[derive(Clone, Debug)]
153pub(super) struct KernelPtConfig {}
154
155// We use the first available PTE bit to mark the frame as tracked.
156// SAFETY: `item_raw_info`, `item_into_raw`, `item_from_raw`, and
157// `item_ref_from_raw` are correctly implemented with respect to the `Item` and
158// `ItemRef` types.
159unsafe impl PageTableConfig for KernelPtConfig {
160    const TOP_LEVEL_INDEX_RANGE: Range<usize> = 256..512;
161    const TOP_LEVEL_CAN_UNMAP: bool = false;
162
163    type E = PageTableEntry;
164    type C = PagingConsts;
165
166    type Item = MappedItem;
167    type ItemRef<'a> = MappedItemRef<'a>;
168
169    fn item_raw_info(item: &Self::Item) -> (Paddr, PagingLevel, PageProperty) {
170        match *item {
171            MappedItem::Tracked(ref frame, mut prop) => {
172                debug_assert!(!prop.priv_flags.contains(PrivilegedPageFlags::AVAIL1));
173                prop.priv_flags |= PrivilegedPageFlags::AVAIL1;
174                let level = frame.map_level();
175                let paddr = frame.paddr();
176                (paddr, level, prop)
177            }
178            MappedItem::Untracked(ref pa, ref level, mut prop) => {
179                debug_assert!(!prop.priv_flags.contains(PrivilegedPageFlags::AVAIL1));
180                prop.priv_flags -= PrivilegedPageFlags::AVAIL1;
181                (*pa, *level, prop)
182            }
183        }
184    }
185
186    unsafe fn item_from_raw(paddr: Paddr, level: PagingLevel, prop: PageProperty) -> Self::Item {
187        if prop.priv_flags.contains(PrivilegedPageFlags::AVAIL1) {
188            debug_assert_eq!(level, 1);
189            // SAFETY: The caller ensures safety.
190            let frame = unsafe { Frame::<dyn AnyFrameMeta>::from_raw(paddr) };
191            MappedItem::Tracked(frame, prop)
192        } else {
193            MappedItem::Untracked(paddr, level, prop)
194        }
195    }
196
197    unsafe fn item_ref_from_raw<'a>(
198        paddr: Paddr,
199        level: PagingLevel,
200        prop: PageProperty,
201    ) -> Self::ItemRef<'a> {
202        if prop.priv_flags.contains(PrivilegedPageFlags::AVAIL1) {
203            debug_assert_eq!(level, 1);
204            // SAFETY: The caller ensures that the frame outlives `'a` and that
205            // the type matches the frame.
206            let frame = unsafe { FrameRef::<dyn AnyFrameMeta>::borrow_paddr(paddr) };
207            MappedItemRef::Tracked(frame, prop)
208        } else {
209            MappedItemRef::Untracked(paddr, level, prop)
210        }
211    }
212}
213
214#[derive(Clone, Debug, Eq, PartialEq)]
215pub(super) enum MappedItem {
216    Tracked(Frame<dyn AnyFrameMeta>, PageProperty),
217    Untracked(Paddr, PagingLevel, PageProperty),
218}
219
220#[derive(Debug)]
221pub(crate) enum MappedItemRef<'a> {
222    #[cfg_attr(not(ktest), expect(dead_code))]
223    Tracked(FrameRef<'a, dyn AnyFrameMeta>, PageProperty),
224    #[cfg_attr(not(ktest), expect(dead_code))]
225    Untracked(Paddr, PagingLevel, PageProperty),
226}
227
228/// Initializes the kernel page table.
229///
230/// This function should be called after:
231///  - the page allocator and the heap allocator are initialized;
232///  - the memory regions are initialized.
233///
234/// This function should be called before:
235///  - any initializer that modifies the kernel page table.
236pub(crate) fn init_kernel_page_table(meta_pages: Segment<MetaPageMeta>) {
237    info!("Initializing the kernel page table");
238
239    // Start to initialize the kernel page table.
240    let kpt = PageTable::<KernelPtConfig>::new_kernel_page_table();
241    let preempt_guard = disable_preempt();
242
243    // In LoongArch64, we don't need to do linear mappings for the kernel because of DMW0.
244    #[cfg(not(target_arch = "loongarch64"))]
245    // Do linear mappings for the kernel.
246    {
247        let max_paddr = crate::mm::frame::max_paddr();
248        let from = LINEAR_MAPPING_BASE_VADDR..LINEAR_MAPPING_BASE_VADDR + max_paddr;
249        let prop = PageProperty {
250            flags: PageFlags::RW,
251            cache: CachePolicy::Writeback,
252            priv_flags: PrivilegedPageFlags::GLOBAL,
253        };
254        let min_level = max_page_level::<KernelPtConfig>(from.len());
255        let mut cursor = kpt
256            .cursor_mut_with_min_level(&preempt_guard, &from, min_level)
257            .unwrap();
258        for (pa, level) in largest_pages::<KernelPtConfig>(from.start, 0, max_paddr) {
259            // SAFETY: We are doing the linear mapping for the kernel.
260            unsafe { cursor.map(MappedItem::Untracked(pa, level, prop)) };
261        }
262    }
263
264    // Map the metadata pages.
265    {
266        let start_va = mapping::frame_to_meta::<PagingConsts>(crate::mm::frame::min_paddr());
267        let from = start_va..start_va + meta_pages.size();
268        let prop = PageProperty {
269            flags: PageFlags::RW,
270            cache: CachePolicy::Writeback,
271            priv_flags: PrivilegedPageFlags::GLOBAL,
272        };
273        let min_level = max_page_level::<KernelPtConfig>(from.len());
274        let mut cursor = kpt
275            .cursor_mut_with_min_level(&preempt_guard, &from, min_level)
276            .unwrap();
277        // We use untracked mapping so that we can benefit from huge pages.
278        // We won't unmap them anyway, so there's no leaking problem yet.
279        let pa_range = meta_pages.into_raw();
280        for (pa, level) in
281            largest_pages::<KernelPtConfig>(from.start, pa_range.start, pa_range.len())
282        {
283            // SAFETY: We are doing the metadata mappings for the kernel.
284            unsafe { cursor.map(MappedItem::Untracked(pa, level, prop)) };
285        }
286    }
287
288    // In LoongArch64, we don't need to do linear mappings for the kernel code because of DMW0.
289    #[cfg(not(target_arch = "loongarch64"))]
290    // Map for the kernel code itself.
291    // TODO: set separated permissions for each segments in the kernel.
292    {
293        let regions = &crate::boot::EARLY_INFO.get().unwrap().memory_regions;
294        let region = regions
295            .iter()
296            .find(|r| r.typ() == MemoryRegionType::Kernel)
297            .unwrap();
298        let offset = kernel_loaded_offset();
299        let from = region.base() + offset..region.end() + offset;
300        let prop = PageProperty {
301            flags: PageFlags::RWX,
302            cache: CachePolicy::Writeback,
303            priv_flags: PrivilegedPageFlags::GLOBAL,
304        };
305        let min_level = max_page_level::<KernelPtConfig>(from.len());
306        let mut cursor = kpt
307            .cursor_mut_with_min_level(&preempt_guard, &from, min_level)
308            .unwrap();
309        for (pa, level) in largest_pages::<KernelPtConfig>(from.start, region.base(), from.len()) {
310            // SAFETY: We are doing the kernel code mapping.
311            unsafe { cursor.map(MappedItem::Untracked(pa, level, prop)) };
312        }
313    }
314
315    KERNEL_PAGE_TABLE.call_once(|| kpt);
316}
317
318/// Activates the kernel page table.
319///
320/// All address translation of symbols in the boot sections must be manually
321/// done from now on.
322///
323/// # Safety
324///
325/// This function must only be called once per CPU.
326pub(crate) unsafe fn activate_kernel_page_table() {
327    let kpt = KERNEL_PAGE_TABLE
328        .get()
329        .expect("The kernel page table is not initialized yet");
330    // SAFETY: The kernel page table is initialized properly.
331    unsafe {
332        kpt.first_activate_unchecked();
333        crate::arch::mm::tlb_flush_all_including_global();
334    }
335}