ostd/arch/x86/trap/
mod.rs

1// SPDX-License-Identifier: MPL-2.0 OR MIT
2//
3// The original source code is from [trapframe-rs](https://github.com/rcore-os/trapframe-rs),
4// which is released under the following license:
5//
6// SPDX-License-Identifier: MIT
7//
8// Copyright (c) 2020 - 2024 Runji Wang
9//
10// We make the following new changes:
11// * Implement the `trap_handler` of Asterinas.
12//
13// These changes are released under the following license:
14//
15// SPDX-License-Identifier: MPL-2.0
16
17//! Handles trap.
18
19pub(super) mod gdt;
20mod idt;
21mod syscall;
22
23use cfg_if::cfg_if;
24use spin::Once;
25
26use super::cpu::context::GeneralRegs;
27use crate::{
28    arch::{
29        cpu::context::CpuException,
30        irq::{HwIrqLine, disable_local, enable_local},
31    },
32    cpu::PrivilegeLevel,
33    ex_table::ExTable,
34    irq::call_irq_callback_functions,
35    mm::MAX_USERSPACE_VADDR,
36};
37
38cfg_if! {
39    if #[cfg(feature = "cvm_guest")] {
40        use tdx_guest::{tdcall, handle_virtual_exception};
41        use crate::arch::tdx_guest::TrapFrameWrapper;
42    }
43}
44
45/// Trap frame of kernel interrupt
46///
47/// # Trap handler
48///
49/// You need to define a handler function like this:
50///
51/// ```
52/// // SAFETY: The name does not collide with other symbols.
53/// #[unsafe(no_mangle)]
54/// extern "sysv64" fn trap_handler(tf: &mut TrapFrame) {
55///     match tf.trap_num {
56///         3 => {
57///             println!("TRAP: BreakPoint");
58///             tf.rip += 1;
59///         }
60///         _ => panic!("TRAP: {:#x?}", tf),
61///     }
62/// }
63/// ```
64#[expect(missing_docs)]
65#[repr(C)]
66#[derive(Clone, Copy, Debug, Default)]
67pub struct TrapFrame {
68    // Pushed by 'trap.S'
69    pub rax: usize,
70    pub rbx: usize,
71    pub rcx: usize,
72    pub rdx: usize,
73    pub rsi: usize,
74    pub rdi: usize,
75    pub rbp: usize,
76    pub r8: usize,
77    pub r9: usize,
78    pub r10: usize,
79    pub r11: usize,
80    pub r12: usize,
81    pub r13: usize,
82    pub r14: usize,
83    pub r15: usize,
84
85    pub trap_num: usize,
86    pub error_code: usize,
87
88    // Pushed by CPU
89    pub rip: usize,
90    pub cs: usize,
91    pub rflags: usize,
92    pub rsp: usize,
93    pub ss: usize,
94}
95
96// Be careful: This assertion is a **soundness** requirement.
97//
98// According to the System V AMD64 ABI, the stack pointer should be aligned to
99// at least 16 bytes. The hardware will align the stack pointer to a 16-byte
100// boundary for exceptions and interrupts ("In IA-32e mode, the RSP is aligned
101// to a 16-byte boundary before pushing the stack frame"), so we only need to
102// ensure the size of a `TrapFrame` is also aligned.
103crate::const_assert!(size_of::<TrapFrame>().is_multiple_of(16));
104
105/// Initializes interrupt handling on x86_64.
106///
107/// This function will:
108/// - Switch to a new, CPU-local [GDT].
109/// - Switch to a new, CPU-local [TSS].
110/// - Switch to a new, global [IDT].
111/// - Enable the [`syscall`] instruction.
112///
113/// [GDT]: https://wiki.osdev.org/GDT
114/// [IDT]: https://wiki.osdev.org/IDT
115/// [TSS]: https://wiki.osdev.org/Task_State_Segment
116/// [`syscall`]: https://www.felixcloutier.com/x86/syscall
117///
118/// # Safety
119///
120/// On the current CPU, this function must be called
121/// - only once and
122/// - before any trap can occur.
123pub(crate) unsafe fn init_on_cpu() {
124    // SAFETY: Since there's no traps, no preemption can occur.
125    unsafe { gdt::init_on_cpu() };
126
127    idt::init_on_cpu();
128
129    // SAFETY: `gdt::init_on_cpu` has been called before.
130    unsafe { syscall::init_on_cpu() };
131}
132
133/// Userspace context.
134#[repr(C)]
135#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
136pub(super) struct RawUserContext {
137    pub(super) general: GeneralRegs,
138    pub(super) trap_num: usize,
139    pub(super) error_code: usize,
140}
141
142/// Handle traps (only from kernel).
143// SAFETY: The name does not collide with other symbols.
144#[unsafe(no_mangle)]
145unsafe extern "sysv64" fn trap_handler(f: &mut TrapFrame) {
146    fn enable_local_if(cond: bool) {
147        if cond {
148            enable_local();
149        }
150    }
151
152    fn disable_local_if(cond: bool) {
153        if cond {
154            disable_local();
155        }
156    }
157
158    // The IRQ state before trapping. We need to ensure that the IRQ state
159    // during exception handling is consistent with the state before the trap.
160    let was_irq_enabled =
161        f.rflags as u64 & x86_64::registers::rflags::RFlags::INTERRUPT_FLAG.bits() > 0;
162
163    let cpu_exception = CpuException::new(f.trap_num, f.error_code);
164    match cpu_exception {
165        #[cfg(feature = "cvm_guest")]
166        Some(CpuException::VirtualizationException) => {
167            let ve_info = tdcall::get_veinfo().expect("#VE handler: fail to get VE info\n");
168            // We need to enable interrupts only after `tdcall::get_veinfo` is called
169            // to avoid nested `#VE`s.
170            enable_local_if(was_irq_enabled);
171            let mut trapframe_wrapper = TrapFrameWrapper(&mut *f);
172            handle_virtual_exception(&mut trapframe_wrapper, &ve_info);
173            *f = *trapframe_wrapper.0;
174            disable_local_if(was_irq_enabled);
175        }
176        Some(CpuException::PageFault(raw_page_fault_info)) => {
177            enable_local_if(was_irq_enabled);
178            // The actual user space implementation should be responsible
179            // for providing mechanism to treat the 0 virtual address.
180            if (0..MAX_USERSPACE_VADDR).contains(&raw_page_fault_info.addr) {
181                handle_user_page_fault(f, cpu_exception.as_ref().unwrap());
182            } else {
183                panic!(
184                    "Cannot handle kernel page fault: {:#x?}; trapframe: {:#x?}",
185                    raw_page_fault_info, f
186                );
187            }
188            disable_local_if(was_irq_enabled);
189        }
190        Some(exception) => {
191            enable_local_if(was_irq_enabled);
192            panic!(
193                "Cannot handle kernel CPU exception: {:#x?}; trapframe: {:#x?}",
194                exception, f
195            );
196        }
197        None => {
198            call_irq_callback_functions(
199                f,
200                &HwIrqLine::new(f.trap_num as u8),
201                PrivilegeLevel::Kernel,
202            );
203        }
204    }
205}
206
207#[expect(clippy::type_complexity)]
208static USER_PAGE_FAULT_HANDLER: Once<fn(&CpuException) -> Result<(), ()>> = Once::new();
209
210/// Injects a custom handler for page faults that occur in the kernel and
211/// are caused by user-space address.
212pub fn inject_user_page_fault_handler(handler: fn(info: &CpuException) -> Result<(), ()>) {
213    USER_PAGE_FAULT_HANDLER.call_once(|| handler);
214}
215
216/// Handles page fault from user space.
217fn handle_user_page_fault(f: &mut TrapFrame, exception: &CpuException) {
218    let handler = USER_PAGE_FAULT_HANDLER
219        .get()
220        .expect("a page fault handler is missing");
221
222    let res = handler(exception);
223    // Copying bytes by bytes can recover directly
224    // if handling the page fault successfully.
225    if res.is_ok() {
226        return;
227    }
228
229    // Use the exception table to recover to normal execution.
230    if let Some(addr) = ExTable::find_recovery_inst_addr(f.rip) {
231        f.rip = addr;
232    } else {
233        panic!("Cannot handle user page fault; trapframe: {:#x?}", f);
234    }
235}
236
237/// User-space code segment selector value.
238pub const USER_CS_VALUE: usize = gdt::USER_CS.0 as usize;
239/// User-space stack segment selector value.
240pub const USER_SS_VALUE: usize = gdt::USER_SS.0 as usize;
ostd/arch/x86/trap/mod.rs

ostd/arch/x86/trap/
mod.rs
