ostd/arch/x86/trap/
mod.rs

1// SPDX-License-Identifier: MPL-2.0 OR MIT
2//
3// The original source code is from [trapframe-rs](https://github.com/rcore-os/trapframe-rs),
4// which is released under the following license:
5//
6// SPDX-License-Identifier: MIT
7//
8// Copyright (c) 2020 - 2024 Runji Wang
9//
10// We make the following new changes:
11// * Implement the `trap_handler` of Asterinas.
12//
13// These changes are released under the following license:
14//
15// SPDX-License-Identifier: MPL-2.0
16
17//! Handles trap.
18
19pub(super) mod gdt;
20mod idt;
21mod syscall;
22
23use cfg_if::cfg_if;
24use spin::Once;
25
26use super::cpu::context::GeneralRegs;
27use crate::{
28    arch::{
29        cpu::context::CpuException,
30        irq::{HwIrqLine, disable_local, enable_local},
31    },
32    cpu::PrivilegeLevel,
33    ex_table::ExTable,
34    irq::call_irq_callback_functions,
35    mm::MAX_USERSPACE_VADDR,
36};
37
38cfg_if! {
39    if #[cfg(feature = "cvm_guest")] {
40        use tdx_guest::{tdcall, handle_virtual_exception};
41        use crate::arch::tdx_guest::TrapFrameWrapper;
42    }
43}
44
45/// Trap frame of kernel interrupt
46///
47/// # Trap handler
48///
49/// You need to define a handler function like this:
50///
51/// ```
52/// // SAFETY: The name does not collide with other symbols.
53/// #[unsafe(no_mangle)]
54/// extern "sysv64" fn trap_handler(tf: &mut TrapFrame) {
55///     match tf.trap_num {
56///         3 => {
57///             println!("TRAP: BreakPoint");
58///             tf.rip += 1;
59///         }
60///         _ => panic!("TRAP: {:#x?}", tf),
61///     }
62/// }
63/// ```
64#[derive(Debug, Default, Clone, Copy)]
65#[repr(C)]
66#[expect(missing_docs)]
67pub struct TrapFrame {
68    // Pushed by 'trap.S'
69    pub rax: usize,
70    pub rbx: usize,
71    pub rcx: usize,
72    pub rdx: usize,
73    pub rsi: usize,
74    pub rdi: usize,
75    pub rbp: usize,
76    pub rsp: usize,
77    pub r8: usize,
78    pub r9: usize,
79    pub r10: usize,
80    pub r11: usize,
81    pub r12: usize,
82    pub r13: usize,
83    pub r14: usize,
84    pub r15: usize,
85    pub _pad: usize,
86
87    pub trap_num: usize,
88    pub error_code: usize,
89
90    // Pushed by CPU
91    pub rip: usize,
92    pub cs: usize,
93    pub rflags: usize,
94}
95
96/// Initializes interrupt handling on x86_64.
97///
98/// This function will:
99/// - Switch to a new, CPU-local [GDT].
100/// - Switch to a new, CPU-local [TSS].
101/// - Switch to a new, global [IDT].
102/// - Enable the [`syscall`] instruction.
103///
104/// [GDT]: https://wiki.osdev.org/GDT
105/// [IDT]: https://wiki.osdev.org/IDT
106/// [TSS]: https://wiki.osdev.org/Task_State_Segment
107/// [`syscall`]: https://www.felixcloutier.com/x86/syscall
108///
109/// # Safety
110///
111/// On the current CPU, this function must be called
112/// - only once and
113/// - before any trap can occur.
114pub(crate) unsafe fn init_on_cpu() {
115    // SAFETY: Since there's no traps, no preemption can occur.
116    unsafe { gdt::init_on_cpu() };
117
118    idt::init_on_cpu();
119
120    // SAFETY: `gdt::init_on_cpu` has been called before.
121    unsafe { syscall::init_on_cpu() };
122}
123
124/// Userspace context.
125#[derive(Debug, Default, Clone, Copy, Eq, PartialEq)]
126#[repr(C)]
127pub(super) struct RawUserContext {
128    pub(super) general: GeneralRegs,
129    pub(super) trap_num: usize,
130    pub(super) error_code: usize,
131}
132
133/// Handle traps (only from kernel).
134// SAFETY: The name does not collide with other symbols.
135#[unsafe(no_mangle)]
136unsafe extern "sysv64" fn trap_handler(f: &mut TrapFrame) {
137    fn enable_local_if(cond: bool) {
138        if cond {
139            enable_local();
140        }
141    }
142
143    fn disable_local_if(cond: bool) {
144        if cond {
145            disable_local();
146        }
147    }
148
149    // The IRQ state before trapping. We need to ensure that the IRQ state
150    // during exception handling is consistent with the state before the trap.
151    let was_irq_enabled =
152        f.rflags as u64 & x86_64::registers::rflags::RFlags::INTERRUPT_FLAG.bits() > 0;
153
154    let cpu_exception = CpuException::new(f.trap_num, f.error_code);
155    match cpu_exception {
156        #[cfg(feature = "cvm_guest")]
157        Some(CpuException::VirtualizationException) => {
158            let ve_info = tdcall::get_veinfo().expect("#VE handler: fail to get VE info\n");
159            // We need to enable interrupts only after `tdcall::get_veinfo` is called
160            // to avoid nested `#VE`s.
161            enable_local_if(was_irq_enabled);
162            let mut trapframe_wrapper = TrapFrameWrapper(&mut *f);
163            handle_virtual_exception(&mut trapframe_wrapper, &ve_info);
164            *f = *trapframe_wrapper.0;
165            disable_local_if(was_irq_enabled);
166        }
167        Some(CpuException::PageFault(raw_page_fault_info)) => {
168            enable_local_if(was_irq_enabled);
169            // The actual user space implementation should be responsible
170            // for providing mechanism to treat the 0 virtual address.
171            if (0..MAX_USERSPACE_VADDR).contains(&raw_page_fault_info.addr) {
172                handle_user_page_fault(f, cpu_exception.as_ref().unwrap());
173            } else {
174                panic!(
175                    "Cannot handle kernel page fault: {:#x?}; trapframe: {:#x?}",
176                    raw_page_fault_info, f
177                );
178            }
179            disable_local_if(was_irq_enabled);
180        }
181        Some(exception) => {
182            enable_local_if(was_irq_enabled);
183            panic!(
184                "Cannot handle kernel CPU exception: {:#x?}; trapframe: {:#x?}",
185                exception, f
186            );
187        }
188        None => {
189            call_irq_callback_functions(
190                f,
191                &HwIrqLine::new(f.trap_num as u8),
192                PrivilegeLevel::Kernel,
193            );
194        }
195    }
196}
197
198#[expect(clippy::type_complexity)]
199static USER_PAGE_FAULT_HANDLER: Once<fn(&CpuException) -> core::result::Result<(), ()>> =
200    Once::new();
201
202/// Injects a custom handler for page faults that occur in the kernel and
203/// are caused by user-space address.
204pub fn inject_user_page_fault_handler(
205    handler: fn(info: &CpuException) -> core::result::Result<(), ()>,
206) {
207    USER_PAGE_FAULT_HANDLER.call_once(|| handler);
208}
209
210/// Handles page fault from user space.
211fn handle_user_page_fault(f: &mut TrapFrame, exception: &CpuException) {
212    let handler = USER_PAGE_FAULT_HANDLER
213        .get()
214        .expect("a page fault handler is missing");
215
216    let res = handler(exception);
217    // Copying bytes by bytes can recover directly
218    // if handling the page fault successfully.
219    if res.is_ok() {
220        return;
221    }
222
223    // Use the exception table to recover to normal execution.
224    if let Some(addr) = ExTable::find_recovery_inst_addr(f.rip) {
225        f.rip = addr;
226    } else {
227        panic!("Cannot handle user page fault; trapframe: {:#x?}", f);
228    }
229}
ostd/arch/x86/trap/mod.rs

ostd/arch/x86/trap/
mod.rs
