1use alloc::boxed::Box;
6use core::arch::x86_64::{_fxrstor64, _fxsave64, _xrstor64, _xsave64};
7
8use bitflags::bitflags;
9use ostd_pod::{FromZeros, IntoBytes};
10use spin::Once;
11use x86::bits64::segmentation::{rdfsbase, rdgsbase, swapgs, wrfsbase, wrgsbase};
12use x86_64::registers::{
13 control::{Cr0, Cr0Flags},
14 rflags::RFlags,
15 xcontrol::XCr0,
16};
17
18use crate::{
19 arch::{
20 irq::HwIrqLine,
21 trap::{RawUserContext, TrapFrame},
22 },
23 cpu::PrivilegeLevel,
24 debug,
25 irq::{DisabledLocalIrqGuard, call_irq_callback_functions},
26 mm::Vaddr,
27 user::{ReturnReason, UserContextApi, UserContextApiInternal, UserModeHooks},
28};
29
30cfg_select! {
31 feature = "cvm_guest" => {
32 mod tdx;
33
34 use tdx::VirtualizationExceptionHandler;
35 }
36}
37
38pub const USER_MODIFIABLE_RFLAGS: usize = (RFlags::CARRY_FLAG.bits()
47 | RFlags::PARITY_FLAG.bits()
48 | RFlags::AUXILIARY_CARRY_FLAG.bits()
49 | RFlags::ZERO_FLAG.bits()
50 | RFlags::SIGN_FLAG.bits()
51 | RFlags::TRAP_FLAG.bits()
52 | RFlags::DIRECTION_FLAG.bits()
53 | RFlags::OVERFLOW_FLAG.bits()
54 | RFlags::RESUME_FLAG.bits()
55 | RFlags::ALIGNMENT_CHECK.bits()
56 | RFlags::NESTED_TASK.bits()) as usize;
57
58#[repr(C)]
60#[derive(Clone, Debug)]
61pub struct UserContext {
62 user_context: RawUserContext,
63 exception: Option<CpuException>,
64}
65
66impl Default for UserContext {
67 fn default() -> Self {
68 const USER_MODE_INITIAL_RFLAGS: usize =
72 (1 << 1) | (RFlags::INTERRUPT_FLAG.bits() | RFlags::ID.bits()) as usize;
73
74 Self {
75 user_context: RawUserContext {
76 general: GeneralRegs::default(),
77 rflags: USER_MODE_INITIAL_RFLAGS,
78 trap_num: 0,
79 error_code: 0,
80 },
81 exception: None,
82 }
83 }
84}
85
86#[expect(missing_docs)]
88#[repr(C)]
89#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
90pub struct GeneralRegs {
91 pub rax: usize,
92 pub rbx: usize,
93 pub rcx: usize,
94 pub rdx: usize,
95 pub rsi: usize,
96 pub rdi: usize,
97 pub rbp: usize,
98 pub rsp: usize,
99 pub r8: usize,
100 pub r9: usize,
101 pub r10: usize,
102 pub r11: usize,
103 pub r12: usize,
104 pub r13: usize,
105 pub r14: usize,
106 pub r15: usize,
107 pub rip: usize,
108}
109
110#[derive(Clone, Copy, Debug, Default)]
112pub struct FsBase(usize);
113
114impl FsBase {
115 pub fn new(addr: usize) -> Self {
117 Self(addr)
118 }
119
120 pub fn addr(&self) -> usize {
122 self.0
123 }
124
125 pub fn save(&mut self) {
127 self.0 = unsafe { rdfsbase() as usize };
129 }
130
131 pub fn load(&self) {
133 unsafe { wrfsbase(self.0 as u64) }
135 }
136}
137
138#[derive(Clone, Copy, Debug, Default)]
140pub struct GsBase(usize);
141
142impl GsBase {
143 pub fn new(addr: usize) -> Self {
145 Self(addr)
146 }
147
148 pub fn addr(&self) -> usize {
150 self.0
151 }
152
153 pub fn save(&mut self, _guard: &DisabledLocalIrqGuard) {
155 unsafe {
159 swapgs();
160 self.0 = rdgsbase() as usize;
161 swapgs();
162 }
163 }
164
165 pub fn load(&self, _guard: &DisabledLocalIrqGuard) {
167 unsafe {
171 swapgs();
172 wrgsbase(self.0 as u64);
173 swapgs();
174 }
175 }
176}
177
178#[derive(Clone, Copy, Debug, Eq, PartialEq)]
195pub enum CpuException {
196 DivisionError,
198 Debug,
200 NonMaskableInterrupt,
202 BreakPoint,
204 Overflow,
206 BoundRangeExceeded,
208 InvalidOpcode,
210 DeviceNotAvailable,
212 DoubleFault,
214 CoprocessorSegmentOverrun,
216 InvalidTss(SelectorErrorCode),
218 SegmentNotPresent(SelectorErrorCode),
220 StackSegmentFault(SelectorErrorCode),
222 GeneralProtectionFault(Option<SelectorErrorCode>),
224 PageFault(RawPageFaultInfo),
226 X87FloatingPointException,
229 AlignmentCheck,
231 MachineCheck,
233 SIMDFloatingPointException,
235 VirtualizationException,
237 ControlProtectionException,
239 HypervisorInjectionException,
242 VMMCommunicationException,
244 SecurityException,
246 Reserved,
249}
250
251impl CpuException {
252 pub(in crate::arch) fn new(trap_num: usize, error_code: usize) -> Option<Self> {
253 let exception = match trap_num {
254 0 => Self::DivisionError,
255 1 => Self::Debug,
256 2 => Self::NonMaskableInterrupt,
257 3 => Self::BreakPoint,
258 4 => Self::Overflow,
259 5 => Self::BoundRangeExceeded,
260 6 => Self::InvalidOpcode,
261 7 => Self::DeviceNotAvailable,
262 8 => {
263 debug_assert_eq!(error_code, 0);
265 Self::DoubleFault
266 }
267 9 => Self::CoprocessorSegmentOverrun,
268 10 => Self::InvalidTss(SelectorErrorCode(error_code)),
269 11 => Self::SegmentNotPresent(SelectorErrorCode(error_code)),
270 12 => Self::StackSegmentFault(SelectorErrorCode(error_code)),
271 13 => {
272 let error_code = if error_code == 0 {
273 None
274 } else {
275 Some(SelectorErrorCode(error_code))
276 };
277 Self::GeneralProtectionFault(error_code)
278 }
279 14 => {
280 let page_fault_addr = x86_64::registers::control::Cr2::read_raw() as usize;
281 Self::PageFault(RawPageFaultInfo {
282 error_code: PageFaultErrorCode::from_bits(error_code).unwrap(),
283 addr: page_fault_addr,
284 })
285 }
286 16 => Self::X87FloatingPointException,
288 17 => Self::AlignmentCheck,
289 18 => Self::MachineCheck,
290 19 => Self::SIMDFloatingPointException,
291 20 => Self::VirtualizationException,
292 21 => Self::ControlProtectionException,
293 28 => Self::HypervisorInjectionException,
295 29 => Self::VMMCommunicationException,
296 30 => Self::SecurityException,
297 15 | 22..=27 | 31 => Self::Reserved,
299 _ => return None,
300 };
301
302 Some(exception)
303 }
304
305 const fn type_(&self) -> CpuExceptionType {
306 match self {
307 Self::Debug => CpuExceptionType::FaultOrTrap,
308 Self::NonMaskableInterrupt => CpuExceptionType::Interrupt,
309 Self::BreakPoint | Self::Overflow => CpuExceptionType::Trap,
310 Self::DoubleFault | Self::MachineCheck => CpuExceptionType::Abort,
311 Self::Reserved => CpuExceptionType::Reserved,
312 _ => CpuExceptionType::Fault,
313 }
314 }
315
316 pub(in crate::arch) const fn is_cpu_exception(trap_num: usize) -> bool {
317 trap_num <= 31
318 }
319}
320
321#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
325pub struct SelectorErrorCode(usize);
326
327impl UserContext {
328 pub fn general_regs(&self) -> &GeneralRegs {
332 &self.user_context.general
333 }
334
335 pub fn general_regs_mut(&mut self) -> &mut GeneralRegs {
337 &mut self.user_context.general
338 }
339
340 pub fn take_exception(&mut self) -> Option<CpuException> {
342 self.exception.take()
343 }
344
345 pub fn rflags(&self) -> usize {
349 self.user_context.rflags
350 }
351
352 pub fn set_rflags(&mut self, rflags: usize) {
357 self.user_context.rflags = (self.user_context.rflags & !USER_MODIFIABLE_RFLAGS)
358 | (rflags & USER_MODIFIABLE_RFLAGS);
359 }
360}
361
362impl UserContextApiInternal for UserContext {
363 fn execute<T: UserModeHooks>(&mut self, hooks: &T) -> ReturnReason {
364 const SYSCALL_TRAPNUM: usize = 0x100;
365
366 loop {
368 crate::task::scheduler::might_preempt();
369
370 let guard = crate::irq::disable_local();
371 hooks.pre_user_run(&guard);
372 self.user_context.run(guard);
373
374 let exception =
375 CpuException::new(self.user_context.trap_num, self.user_context.error_code);
376 match exception {
377 #[cfg(feature = "cvm_guest")]
378 Some(CpuException::VirtualizationException) => {
379 let ve_handler = VirtualizationExceptionHandler::new();
380 crate::arch::irq::enable_local();
383 ve_handler.handle(self);
384 }
385 Some(exception) if exception.type_().is_fault_or_trap() => {
386 crate::arch::irq::enable_local();
387 self.exception = Some(exception);
388 return ReturnReason::UserException;
389 }
390 Some(exception) => {
391 panic!(
392 "Cannot handle user CPU exception: {:?}; trapframe: {:?}",
393 exception,
394 self.as_trap_frame()
395 );
396 }
397 None if self.user_context.trap_num == SYSCALL_TRAPNUM => {
398 crate::arch::irq::enable_local();
399 return ReturnReason::UserSyscall;
400 }
401 None => {
402 call_irq_callback_functions(
403 &self.as_trap_frame(),
404 &HwIrqLine::new(self.as_trap_frame().trap_num as u8),
405 PrivilegeLevel::User,
406 );
407 crate::arch::irq::enable_local();
408 }
409 }
410
411 if hooks.has_kernel_event() {
412 break ReturnReason::KernelEvent;
413 }
414 }
415 }
416
417 fn as_trap_frame(&self) -> TrapFrame {
418 TrapFrame {
419 rax: self.user_context.general.rax,
420 rbx: self.user_context.general.rbx,
421 rcx: self.user_context.general.rcx,
422 rdx: self.user_context.general.rdx,
423 rsi: self.user_context.general.rsi,
424 rdi: self.user_context.general.rdi,
425 rbp: self.user_context.general.rbp,
426 r8: self.user_context.general.r8,
427 r9: self.user_context.general.r9,
428 r10: self.user_context.general.r10,
429 r11: self.user_context.general.r11,
430 r12: self.user_context.general.r12,
431 r13: self.user_context.general.r13,
432 r14: self.user_context.general.r14,
433 r15: self.user_context.general.r15,
434 trap_num: self.user_context.trap_num,
435 error_code: self.user_context.error_code,
436 rip: self.user_context.general.rip,
437 cs: 0,
438 rflags: self.user_context.rflags,
439 rsp: self.user_context.general.rsp,
440 ss: 0,
441 }
442 }
443}
444
445#[derive(Clone, Copy, Debug, Eq, PartialEq)]
455enum CpuExceptionType {
456 Fault,
458 Trap,
460 FaultOrTrap,
462 Interrupt,
464 Abort,
466 Reserved,
468}
469
470impl CpuExceptionType {
471 fn is_fault_or_trap(self) -> bool {
473 match self {
474 CpuExceptionType::Trap | CpuExceptionType::Fault | CpuExceptionType::FaultOrTrap => {
475 true
476 }
477 CpuExceptionType::Abort | CpuExceptionType::Interrupt | CpuExceptionType::Reserved => {
478 false
479 }
480 }
481 }
482}
483
484#[derive(Clone, Copy, Debug, Eq, PartialEq)]
486pub struct RawPageFaultInfo {
487 pub error_code: PageFaultErrorCode,
489 pub addr: Vaddr,
491}
492
493bitflags! {
494 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
496 pub struct PageFaultErrorCode: usize {
497 const PRESENT = 1 << 0;
499 const WRITE = 1 << 1;
501 const USER = 1 << 2;
503 const RESERVED = 1 << 3;
506 const INSTRUCTION = 1 << 4;
508 const PROTECTION = 1 << 5;
511 const SHADOW_STACK = 1 << 6;
513 const HLAT = 1 << 7;
515 const SGX = 1 << 15;
518 }
519}
520
521impl UserContextApi for UserContext {
522 fn set_instruction_pointer(&mut self, ip: usize) {
523 self.set_rip(ip);
524 }
525
526 fn set_stack_pointer(&mut self, sp: usize) {
527 self.set_rsp(sp)
528 }
529
530 fn stack_pointer(&self) -> usize {
531 self.rsp()
532 }
533
534 fn instruction_pointer(&self) -> usize {
535 self.rip()
536 }
537}
538
539macro_rules! cpu_context_impl_getter_setter {
540 ( $( [ $field: ident, $setter_name: ident] ),*) => {
541 impl UserContext {
542 $(
543 #[doc = concat!("Gets the value of ", stringify!($field))]
544 #[inline(always)]
545 pub fn $field(&self) -> usize {
546 self.user_context.general.$field
547 }
548
549 #[doc = concat!("Sets the value of ", stringify!(field))]
550 #[inline(always)]
551 pub fn $setter_name(&mut self, $field: usize) {
552 self.user_context.general.$field = $field;
553 }
554 )*
555 }
556 };
557}
558
559cpu_context_impl_getter_setter!(
560 [rax, set_rax],
561 [rbx, set_rbx],
562 [rcx, set_rcx],
563 [rdx, set_rdx],
564 [rsi, set_rsi],
565 [rdi, set_rdi],
566 [rbp, set_rbp],
567 [rsp, set_rsp],
568 [r8, set_r8],
569 [r9, set_r9],
570 [r10, set_r10],
571 [r11, set_r11],
572 [r12, set_r12],
573 [r13, set_r13],
574 [r14, set_r14],
575 [r15, set_r15],
576 [rip, set_rip]
577);
578
579#[derive(Debug)]
583pub struct FpuContext {
584 xsave_area: Box<XSaveArea>,
585 area_size: usize,
586}
587
588impl FpuContext {
589 pub fn new() -> Self {
591 let mut area_size = size_of::<FxSaveArea>();
592 if let Some(xsave_area_size) = XSAVE_AREA_SIZE.get() {
593 area_size = area_size.max(*xsave_area_size);
594 }
595
596 Self {
597 xsave_area: Box::new(XSaveArea::new()),
598 area_size,
599 }
600 }
601
602 pub fn save(&mut self) {
604 let mem_addr = self.as_bytes_mut().as_mut_ptr();
605
606 if XSTATE_MAX_FEATURES.is_completed() {
607 unsafe { _xsave64(mem_addr, XFEATURE_MASK_USER_RESTORE) };
608 } else {
609 unsafe { _fxsave64(mem_addr) };
610 }
611
612 debug!("Save FPU context");
613 }
614
615 pub fn load(&self) {
617 let mem_addr = self.as_bytes().as_ptr();
618
619 if let Some(xstate_max_features) = XSTATE_MAX_FEATURES.get() {
620 let rs_mask = XFEATURE_MASK_USER_RESTORE & *xstate_max_features;
621
622 unsafe { _xrstor64(mem_addr, rs_mask) };
623 } else {
624 unsafe { _fxrstor64(mem_addr) };
625 }
626
627 debug!("Load FPU context");
628 }
629
630 pub fn as_bytes(&self) -> &[u8] {
632 &self.xsave_area.as_bytes()[..self.area_size]
633 }
634
635 pub fn as_bytes_mut(&mut self) -> &mut [u8] {
637 &mut self.xsave_area.as_mut_bytes()[..self.area_size]
638 }
639}
640
641impl Default for FpuContext {
642 fn default() -> Self {
643 Self::new()
644 }
645}
646
647impl Clone for FpuContext {
648 fn clone(&self) -> Self {
649 let mut xsave_area = Box::new(XSaveArea::new());
650 xsave_area.fxsave_area = self.xsave_area.fxsave_area;
651 xsave_area.features = self.xsave_area.features;
652 xsave_area.compaction = self.xsave_area.compaction;
653 if self.area_size > size_of::<FxSaveArea>() {
654 let len = self.area_size - size_of::<FxSaveArea>() - 64;
655 xsave_area.extended_state_area[..len]
656 .copy_from_slice(&self.xsave_area.extended_state_area[..len]);
657 }
658
659 Self {
660 xsave_area,
661 area_size: self.area_size,
662 }
663 }
664}
665
666#[repr(C)]
668#[repr(align(64))]
669#[derive(Clone, Copy, Debug, Pod)]
670struct XSaveArea {
671 fxsave_area: FxSaveArea,
672 features: u64,
673 compaction: u64,
674 reserved: [u64; 6],
675 extended_state_area: [u8; MAX_XSAVE_AREA_SIZE - size_of::<FxSaveArea>() - 64],
676}
677
678impl XSaveArea {
679 fn new() -> Self {
680 let features = if let Some(xstate_max_features) = XSTATE_MAX_FEATURES.get() {
681 XCr0::read().bits() & *xstate_max_features
682 } else {
683 0
684 };
685
686 let mut xsave_area = Self::new_zeroed();
687 xsave_area.fxsave_area.control = 0x037F;
690 xsave_area.fxsave_area.tag = 0;
697 xsave_area.fxsave_area.mxcsr = 0x1F80;
698 xsave_area.features = features;
699
700 xsave_area
701 }
702}
703
704#[repr(C)]
706#[repr(align(16))]
707#[derive(Clone, Copy, Debug, Pod)]
708struct FxSaveArea {
709 control: u16, status: u16, tag: u8, reserved1: u8, op: u16, ip: u32, cs: u32, dp: u32, ds: u32, mxcsr: u32, mxcsr_mask: u32, st_space: [u32; 32], xmm_space: [u32; 64], reserved2: [u32; 12], reserved3: [u32; 12], }
725
726static XSTATE_MAX_FEATURES: Once<u64> = Once::new();
728
729const XFEATURE_MASK_USER_RESTORE: u64 = 0b1110_0111;
733
734static XSAVE_AREA_SIZE: Once<usize> = Once::new();
736
737const MAX_XSAVE_AREA_SIZE: usize = 4096;
739
740pub(in crate::arch) fn enable_essential_features() {
741 use super::extension::{IsaExtensions, has_extensions};
742
743 if has_extensions(IsaExtensions::XSAVE) {
744 XSTATE_MAX_FEATURES.call_once(|| super::cpuid::query_xstate_max_features().unwrap());
745 XSAVE_AREA_SIZE.call_once(|| {
746 let xsave_area_size = super::cpuid::query_xsave_area_size().unwrap() as usize;
747 assert!(xsave_area_size <= MAX_XSAVE_AREA_SIZE);
748 xsave_area_size
749 });
750 }
751
752 {
755 let mut cr0 = Cr0::read();
756 cr0.remove(Cr0Flags::TASK_SWITCHED | Cr0Flags::EMULATE_COPROCESSOR);
757
758 unsafe {
759 Cr0::write(cr0);
760 core::arch::asm!("fninit");
762 }
763 }
764}