1#![cfg_attr(target_arch = "loongarch64", expect(unused_imports))]
38
39pub(crate) mod kvirt_area;
40
41use core::ops::Range;
42
43use spin::Once;
44
45#[cfg(ktest)]
46mod test;
47
48use super::{
49 Frame, HasSize, Paddr, PagingConstsTrait, Vaddr,
50 frame::{
51 Segment,
52 meta::{AnyFrameMeta, MetaPageMeta, mapping},
53 },
54 page_prop::{CachePolicy, PageFlags, PageProperty, PrivilegedPageFlags},
55 page_table::{PageTable, PageTableConfig, largest_pages, max_page_level},
56};
57use crate::{
58 arch::mm::{PageTableEntry, PagingConsts},
59 boot::memory_region::MemoryRegionType,
60 const_assert, info,
61 mm::{HasPaddr, PAGE_SIZE, PagingLevel, frame::FrameRef},
62 task::disable_preempt,
63};
64
65const_assert!(PagingConsts::ADDRESS_WIDTH >= 39);
69const ADDR_WIDTH_SHIFT: usize = PagingConsts::ADDRESS_WIDTH - 39;
70
71#[cfg(not(target_arch = "loongarch64"))]
73pub(super) const KERNEL_BASE_VADDR: Vaddr = 0xffff_ffc0_0000_0000 << ADDR_WIDTH_SHIFT;
74#[cfg(target_arch = "loongarch64")]
75pub(super) const KERNEL_BASE_VADDR: Vaddr = 0x9000_0000_0000_0000;
76pub(super) const KERNEL_END_VADDR: Vaddr = 0xffff_ffff_ffff_0000;
78
79pub const MAX_USERSPACE_VADDR: Vaddr = (0x0000_0040_0000_0000 << ADDR_WIDTH_SHIFT) - PAGE_SIZE;
89
90pub const KERNEL_VADDR_RANGE: Range<Vaddr> = KERNEL_BASE_VADDR..KERNEL_END_VADDR;
95
96pub(crate) fn kernel_loaded_offset() -> usize {
101 #[cfg(target_arch = "riscv64")]
102 {
103 unsafe extern "C" {
104 static __kernel_loaded_offset: usize;
105 }
106 unsafe { __kernel_loaded_offset }
109 }
110 #[cfg(not(target_arch = "riscv64"))]
111 {
112 KERNEL_CODE_BASE_VADDR
113 }
114}
115
116#[cfg(target_arch = "x86_64")]
117const KERNEL_CODE_BASE_VADDR: usize = 0xffff_ffff_8000_0000;
118#[cfg(target_arch = "aarch64")]
119const KERNEL_CODE_BASE_VADDR: usize = 0xffff_ffff_0000_0000;
120#[cfg(target_arch = "loongarch64")]
121const KERNEL_CODE_BASE_VADDR: usize = 0x9000_0000_0000_0000;
122
123const FRAME_METADATA_CAP_VADDR: Vaddr = 0xffff_fff0_8000_0000 << ADDR_WIDTH_SHIFT;
124const FRAME_METADATA_BASE_VADDR: Vaddr = 0xffff_fff0_0000_0000 << ADDR_WIDTH_SHIFT;
125pub(super) const FRAME_METADATA_RANGE: Range<Vaddr> =
126 FRAME_METADATA_BASE_VADDR..FRAME_METADATA_CAP_VADDR;
127
128const VMALLOC_BASE_VADDR: Vaddr = 0xffff_ffe0_0000_0000 << ADDR_WIDTH_SHIFT;
129pub(super) const VMALLOC_VADDR_RANGE: Range<Vaddr> = VMALLOC_BASE_VADDR..FRAME_METADATA_BASE_VADDR;
130
131#[cfg(not(target_arch = "loongarch64"))]
134pub(crate) const LINEAR_MAPPING_BASE_VADDR: Vaddr = 0xffff_ffc0_0000_0000 << ADDR_WIDTH_SHIFT;
135#[cfg(target_arch = "loongarch64")]
136pub(crate) const LINEAR_MAPPING_BASE_VADDR: Vaddr = 0x9000_0000_0000_0000;
137pub(crate) const LINEAR_MAPPING_VADDR_RANGE: Range<Vaddr> =
138 LINEAR_MAPPING_BASE_VADDR..VMALLOC_BASE_VADDR;
139
140pub(crate) fn paddr_to_vaddr(pa: Paddr) -> usize {
142 debug_assert!(pa < VMALLOC_BASE_VADDR - LINEAR_MAPPING_BASE_VADDR);
143 pa + LINEAR_MAPPING_BASE_VADDR
144}
145
146pub(super) static KERNEL_PAGE_TABLE: Once<PageTable<KernelPtConfig>> = Once::new();
151
152#[derive(Clone, Debug)]
153pub(super) struct KernelPtConfig {}
154
155unsafe impl PageTableConfig for KernelPtConfig {
160 const TOP_LEVEL_INDEX_RANGE: Range<usize> = 256..512;
161 const TOP_LEVEL_CAN_UNMAP: bool = false;
162
163 type E = PageTableEntry;
164 type C = PagingConsts;
165
166 type Item = MappedItem;
167 type ItemRef<'a> = MappedItemRef<'a>;
168
169 fn item_raw_info(item: &Self::Item) -> (Paddr, PagingLevel, PageProperty) {
170 match *item {
171 MappedItem::Tracked(ref frame, mut prop) => {
172 debug_assert!(!prop.priv_flags.contains(PrivilegedPageFlags::AVAIL1));
173 prop.priv_flags |= PrivilegedPageFlags::AVAIL1;
174 let level = frame.map_level();
175 let paddr = frame.paddr();
176 (paddr, level, prop)
177 }
178 MappedItem::Untracked(ref pa, ref level, mut prop) => {
179 debug_assert!(!prop.priv_flags.contains(PrivilegedPageFlags::AVAIL1));
180 prop.priv_flags -= PrivilegedPageFlags::AVAIL1;
181 (*pa, *level, prop)
182 }
183 }
184 }
185
186 unsafe fn item_from_raw(paddr: Paddr, level: PagingLevel, prop: PageProperty) -> Self::Item {
187 if prop.priv_flags.contains(PrivilegedPageFlags::AVAIL1) {
188 debug_assert_eq!(level, 1);
189 let frame = unsafe { Frame::<dyn AnyFrameMeta>::from_raw(paddr) };
191 MappedItem::Tracked(frame, prop)
192 } else {
193 MappedItem::Untracked(paddr, level, prop)
194 }
195 }
196
197 unsafe fn item_ref_from_raw<'a>(
198 paddr: Paddr,
199 level: PagingLevel,
200 prop: PageProperty,
201 ) -> Self::ItemRef<'a> {
202 if prop.priv_flags.contains(PrivilegedPageFlags::AVAIL1) {
203 debug_assert_eq!(level, 1);
204 let frame = unsafe { FrameRef::<dyn AnyFrameMeta>::borrow_paddr(paddr) };
207 MappedItemRef::Tracked(frame, prop)
208 } else {
209 MappedItemRef::Untracked(paddr, level, prop)
210 }
211 }
212}
213
214#[derive(Clone, Debug, Eq, PartialEq)]
215pub(super) enum MappedItem {
216 Tracked(Frame<dyn AnyFrameMeta>, PageProperty),
217 Untracked(Paddr, PagingLevel, PageProperty),
218}
219
220#[derive(Debug)]
221pub(crate) enum MappedItemRef<'a> {
222 #[cfg_attr(not(ktest), expect(dead_code))]
223 Tracked(FrameRef<'a, dyn AnyFrameMeta>, PageProperty),
224 #[cfg_attr(not(ktest), expect(dead_code))]
225 Untracked(Paddr, PagingLevel, PageProperty),
226}
227
228pub(crate) fn init_kernel_page_table(meta_pages: Segment<MetaPageMeta>) {
237 info!("Initializing the kernel page table");
238
239 let kpt = PageTable::<KernelPtConfig>::new_kernel_page_table();
241 let preempt_guard = disable_preempt();
242
243 #[cfg(not(target_arch = "loongarch64"))]
245 {
247 let max_paddr = crate::mm::frame::max_paddr();
248 let from = LINEAR_MAPPING_BASE_VADDR..LINEAR_MAPPING_BASE_VADDR + max_paddr;
249 let prop = PageProperty {
250 flags: PageFlags::RW,
251 cache: CachePolicy::Writeback,
252 priv_flags: PrivilegedPageFlags::GLOBAL,
253 };
254 let min_level = max_page_level::<KernelPtConfig>(from.len());
255 let mut cursor = kpt
256 .cursor_mut_with_min_level(&preempt_guard, &from, min_level)
257 .unwrap();
258 for (pa, level) in largest_pages::<KernelPtConfig>(from.start, 0, max_paddr) {
259 unsafe { cursor.map(MappedItem::Untracked(pa, level, prop)) };
261 }
262 }
263
264 {
266 let start_va = mapping::frame_to_meta::<PagingConsts>(crate::mm::frame::min_paddr());
267 let from = start_va..start_va + meta_pages.size();
268 let prop = PageProperty {
269 flags: PageFlags::RW,
270 cache: CachePolicy::Writeback,
271 priv_flags: PrivilegedPageFlags::GLOBAL,
272 };
273 let min_level = max_page_level::<KernelPtConfig>(from.len());
274 let mut cursor = kpt
275 .cursor_mut_with_min_level(&preempt_guard, &from, min_level)
276 .unwrap();
277 let pa_range = meta_pages.into_raw();
280 for (pa, level) in
281 largest_pages::<KernelPtConfig>(from.start, pa_range.start, pa_range.len())
282 {
283 unsafe { cursor.map(MappedItem::Untracked(pa, level, prop)) };
285 }
286 }
287
288 #[cfg(not(target_arch = "loongarch64"))]
290 {
293 let regions = &crate::boot::EARLY_INFO.get().unwrap().memory_regions;
294 let region = regions
295 .iter()
296 .find(|r| r.typ() == MemoryRegionType::Kernel)
297 .unwrap();
298 let offset = kernel_loaded_offset();
299 let from = region.base() + offset..region.end() + offset;
300 let prop = PageProperty {
301 flags: PageFlags::RWX,
302 cache: CachePolicy::Writeback,
303 priv_flags: PrivilegedPageFlags::GLOBAL,
304 };
305 let min_level = max_page_level::<KernelPtConfig>(from.len());
306 let mut cursor = kpt
307 .cursor_mut_with_min_level(&preempt_guard, &from, min_level)
308 .unwrap();
309 for (pa, level) in largest_pages::<KernelPtConfig>(from.start, region.base(), from.len()) {
310 unsafe { cursor.map(MappedItem::Untracked(pa, level, prop)) };
312 }
313 }
314
315 KERNEL_PAGE_TABLE.call_once(|| kpt);
316}
317
318pub(crate) unsafe fn activate_kernel_page_table() {
327 let kpt = KERNEL_PAGE_TABLE
328 .get()
329 .expect("The kernel page table is not initialized yet");
330 unsafe {
332 kpt.first_activate_unchecked();
333 crate::arch::mm::tlb_flush_all_including_global();
334 }
335}