ostd/arch/x86/cpu/context/
mod.rs

1// SPDX-License-Identifier: MPL-2.0
2
3//! CPU execution context control.
4
5use alloc::boxed::Box;
6use core::arch::x86_64::{_fxrstor64, _fxsave64, _xrstor64, _xsave64};
7
8use bitflags::bitflags;
9use ostd_pod::{FromZeros, IntoBytes};
10use spin::Once;
11use x86::bits64::segmentation::{rdfsbase, rdgsbase, swapgs, wrfsbase, wrgsbase};
12use x86_64::registers::{
13    control::{Cr0, Cr0Flags},
14    rflags::RFlags,
15    xcontrol::XCr0,
16};
17
18use crate::{
19    arch::{
20        irq::HwIrqLine,
21        trap::{RawUserContext, TrapFrame},
22    },
23    cpu::PrivilegeLevel,
24    debug,
25    irq::{DisabledLocalIrqGuard, call_irq_callback_functions},
26    mm::Vaddr,
27    user::{ReturnReason, UserContextApi, UserContextApiInternal, UserModeHooks},
28};
29
30cfg_select! {
31    feature = "cvm_guest" => {
32        mod tdx;
33
34        use tdx::VirtualizationExceptionHandler;
35    }
36}
37
38/// Userspace CPU context, including general-purpose registers and exception information.
39#[repr(C)]
40#[derive(Clone, Debug, Default)]
41pub struct UserContext {
42    user_context: RawUserContext,
43    exception: Option<CpuException>,
44}
45
46/// General registers.
47#[expect(missing_docs)]
48#[repr(C)]
49#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
50pub struct GeneralRegs {
51    pub rax: usize,
52    pub rbx: usize,
53    pub rcx: usize,
54    pub rdx: usize,
55    pub rsi: usize,
56    pub rdi: usize,
57    pub rbp: usize,
58    pub rsp: usize,
59    pub r8: usize,
60    pub r9: usize,
61    pub r10: usize,
62    pub r11: usize,
63    pub r12: usize,
64    pub r13: usize,
65    pub r14: usize,
66    pub r15: usize,
67    pub rip: usize,
68    pub rflags: usize,
69}
70
71/// The user-mode FS base register.
72#[derive(Clone, Copy, Debug, Default)]
73pub struct FsBase(usize);
74
75impl FsBase {
76    /// Creates a new `FsBase` with the given address.
77    pub fn new(addr: usize) -> Self {
78        Self(addr)
79    }
80
81    /// Returns the stored address.
82    pub fn addr(&self) -> usize {
83        self.0
84    }
85
86    /// Saves the current CPU FS base into this struct.
87    pub fn save(&mut self) {
88        // SAFETY: Reading the user FS base does not affect kernel code.
89        self.0 = unsafe { rdfsbase() as usize };
90    }
91
92    /// Loads this struct's FS base onto the CPU.
93    pub fn load(&self) {
94        // SAFETY: Writing the user FS base does not affect kernel code.
95        unsafe { wrfsbase(self.0 as u64) }
96    }
97}
98
99/// The user-mode GS base register.
100#[derive(Clone, Copy, Debug, Default)]
101pub struct GsBase(usize);
102
103impl GsBase {
104    /// Creates a new `GsBase` with the given address.
105    pub fn new(addr: usize) -> Self {
106        Self(addr)
107    }
108
109    /// Returns the stored address.
110    pub fn addr(&self) -> usize {
111        self.0
112    }
113
114    /// Saves the current CPU GS base into this struct.
115    pub fn save(&mut self, _guard: &DisabledLocalIrqGuard) {
116        // SAFETY:
117        // 1. In these steps, we have disabled the IRQ and are not using the kernel GS base.
118        // 2. Reading the user GS base does not affect kernel code.
119        unsafe {
120            swapgs();
121            self.0 = rdgsbase() as usize;
122            swapgs();
123        }
124    }
125
126    /// Loads this struct's GS base onto the CPU.
127    pub fn load(&self, _guard: &DisabledLocalIrqGuard) {
128        // SAFETY:
129        // 1. In these steps, we have disabled the IRQ and are not using the kernel GS base.
130        // 2. Writing the user GS base does not affect kernel code.
131        unsafe {
132            swapgs();
133            wrgsbase(self.0 as u64);
134            swapgs();
135        }
136    }
137}
138
139/// Architectural CPU exceptions (x86-64 vectors 0-31).
140///
141/// For the authoritative specification of each vector, see the
142/// Intel® 64 and IA-32 Architectures Software Developer’s Manual,
143/// Volume 3 “System Programming Guide”, Chapter 6 “Interrupt and Exception
144/// Handling”, in particular Section 6.15 “Exception and Interrupt
145/// Reference”.
146///
147/// Every enum variant corresponds to one exception defined by the
148/// Intel/AMD architecture.
149/// Variants that naturally carry an error code (or other error information)
150/// expose it through their associated data fields.
151//
152// TODO: Some exceptions (like `AlignmentCheck`) also push an
153//       error code onto the stack, but that detail is not yet represented
154//       in this type definition.
155#[derive(Clone, Copy, Debug, Eq, PartialEq)]
156pub enum CpuException {
157    ///  0 – #DE  Divide-by-zero error.
158    DivisionError,
159    ///  1 – #DB  Debug.
160    Debug,
161    ///  2 – NMI  Non-maskable interrupt.
162    NonMaskableInterrupt,
163    ///  3 – #BP  Breakpoint (INT3).
164    BreakPoint,
165    ///  4 – #OF  Overflow.
166    Overflow,
167    ///  5 – #BR  Bound-range exceeded.
168    BoundRangeExceeded,
169    ///  6 – #UD  Invalid or undefined opcode.
170    InvalidOpcode,
171    ///  7 – #NM  Device not available (FPU/MMX/SSE disabled).
172    DeviceNotAvailable,
173    ///  8 – #DF  Double fault (always pushes an error code of 0).
174    DoubleFault,
175    ///  9 – Coprocessor segment overrun (reserved on modern CPUs).
176    CoprocessorSegmentOverrun,
177    /// 10 – #TS  Invalid TSS.
178    InvalidTss(SelectorErrorCode),
179    /// 11 – #NP  Segment not present.
180    SegmentNotPresent(SelectorErrorCode),
181    /// 12 – #SS  Stack-segment fault.
182    StackSegmentFault(SelectorErrorCode),
183    /// 13 – #GP  General protection fault
184    GeneralProtectionFault(Option<SelectorErrorCode>),
185    /// 14 – #PF  Page fault.
186    PageFault(RawPageFaultInfo),
187    // 15: Reserved
188    /// 16 – #MF  x87 floating-point exception.
189    X87FloatingPointException,
190    /// 17 – #AC  Alignment check.
191    AlignmentCheck,
192    /// 18 – #MC  Machine check.
193    MachineCheck,
194    /// 19 – #XM / #XF  SIMD/FPU floating-point exception.
195    SIMDFloatingPointException,
196    /// 20 – #VE  Virtualization exception.
197    VirtualizationException,
198    /// 21 – #CP  Control protection exception (CET).
199    ControlProtectionException,
200    // 22-27: Reserved
201    /// 28 – #HV  Hypervisor injection exception.
202    HypervisorInjectionException,
203    /// 29 – #VC  VMM communication exception (SEV-ES GHCB).
204    VMMCommunicationException,
205    /// 30 – #SX  Security exception.
206    SecurityException,
207    // 31: Reserved
208    /// Catch-all for reserved or undefined vector numbers.
209    Reserved,
210}
211
212impl CpuException {
213    pub(crate) fn new(trap_num: usize, error_code: usize) -> Option<Self> {
214        let exception = match trap_num {
215            0 => Self::DivisionError,
216            1 => Self::Debug,
217            2 => Self::NonMaskableInterrupt,
218            3 => Self::BreakPoint,
219            4 => Self::Overflow,
220            5 => Self::BoundRangeExceeded,
221            6 => Self::InvalidOpcode,
222            7 => Self::DeviceNotAvailable,
223            8 => {
224                // A double fault will always generate an error code with a value of zero.
225                debug_assert_eq!(error_code, 0);
226                Self::DoubleFault
227            }
228            9 => Self::CoprocessorSegmentOverrun,
229            10 => Self::InvalidTss(SelectorErrorCode(error_code)),
230            11 => Self::SegmentNotPresent(SelectorErrorCode(error_code)),
231            12 => Self::StackSegmentFault(SelectorErrorCode(error_code)),
232            13 => {
233                let error_code = if error_code == 0 {
234                    None
235                } else {
236                    Some(SelectorErrorCode(error_code))
237                };
238                Self::GeneralProtectionFault(error_code)
239            }
240            14 => {
241                let page_fault_addr = x86_64::registers::control::Cr2::read_raw() as usize;
242                Self::PageFault(RawPageFaultInfo {
243                    error_code: PageFaultErrorCode::from_bits(error_code).unwrap(),
244                    addr: page_fault_addr,
245                })
246            }
247            // Reserved 15
248            16 => Self::X87FloatingPointException,
249            17 => Self::AlignmentCheck,
250            18 => Self::MachineCheck,
251            19 => Self::SIMDFloatingPointException,
252            20 => Self::VirtualizationException,
253            21 => Self::ControlProtectionException,
254            // Reserved 22-27
255            28 => Self::HypervisorInjectionException,
256            29 => Self::VMMCommunicationException,
257            30 => Self::SecurityException,
258            // Reserved 31
259            15 | 22..=27 | 31 => Self::Reserved,
260            _ => return None,
261        };
262
263        Some(exception)
264    }
265
266    const fn type_(&self) -> CpuExceptionType {
267        match self {
268            Self::Debug => CpuExceptionType::FaultOrTrap,
269            Self::NonMaskableInterrupt => CpuExceptionType::Interrupt,
270            Self::BreakPoint | Self::Overflow => CpuExceptionType::Trap,
271            Self::DoubleFault | Self::MachineCheck => CpuExceptionType::Abort,
272            Self::Reserved => CpuExceptionType::Reserved,
273            _ => CpuExceptionType::Fault,
274        }
275    }
276
277    pub(crate) const fn is_cpu_exception(trap_num: usize) -> bool {
278        trap_num <= 31
279    }
280}
281
282/// Selector error code.
283///
284/// Reference: <https://wiki.osdev.org/Exceptions#Selector_Error_Code>.
285#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
286pub struct SelectorErrorCode(usize);
287
288impl UserContext {
289    /// Returns a reference to the general registers.
290    pub fn general_regs(&self) -> &GeneralRegs {
291        &self.user_context.general
292    }
293
294    /// Returns a mutable reference to the general registers
295    pub fn general_regs_mut(&mut self) -> &mut GeneralRegs {
296        &mut self.user_context.general
297    }
298
299    /// Takes the CPU exception out.
300    pub fn take_exception(&mut self) -> Option<CpuException> {
301        self.exception.take()
302    }
303}
304
305impl UserContextApiInternal for UserContext {
306    fn execute<T: UserModeHooks>(&mut self, hooks: &T) -> ReturnReason {
307        // set interrupt flag so that in user mode it can receive external interrupts
308        // set ID flag which means cpu support CPUID instruction
309        self.user_context.general.rflags |= (RFlags::INTERRUPT_FLAG | RFlags::ID).bits() as usize;
310
311        const SYSCALL_TRAPNUM: usize = 0x100;
312
313        // Return when it is syscall or cpu exception type is Fault or Trap.
314        loop {
315            crate::task::scheduler::might_preempt();
316
317            let guard = crate::irq::disable_local();
318            hooks.pre_user_run(&guard);
319            self.user_context.run(guard);
320
321            let exception =
322                CpuException::new(self.user_context.trap_num, self.user_context.error_code);
323            match exception {
324                #[cfg(feature = "cvm_guest")]
325                Some(CpuException::VirtualizationException) => {
326                    let ve_handler = VirtualizationExceptionHandler::new();
327                    // Check out the doc of `VirtualizationExceptionHandler::new` to
328                    // see why IRQs must enabled _after_ instantiating a `VirtualizationExceptionHandler`.
329                    crate::arch::irq::enable_local();
330                    ve_handler.handle(self);
331                }
332                Some(exception) if exception.type_().is_fault_or_trap() => {
333                    crate::arch::irq::enable_local();
334                    self.exception = Some(exception);
335                    return ReturnReason::UserException;
336                }
337                Some(exception) => {
338                    panic!(
339                        "Cannot handle user CPU exception: {:?}; trapframe: {:?}",
340                        exception,
341                        self.as_trap_frame()
342                    );
343                }
344                None if self.user_context.trap_num == SYSCALL_TRAPNUM => {
345                    crate::arch::irq::enable_local();
346                    return ReturnReason::UserSyscall;
347                }
348                None => {
349                    call_irq_callback_functions(
350                        &self.as_trap_frame(),
351                        &HwIrqLine::new(self.as_trap_frame().trap_num as u8),
352                        PrivilegeLevel::User,
353                    );
354                    crate::arch::irq::enable_local();
355                }
356            }
357
358            if hooks.has_kernel_event() {
359                break ReturnReason::KernelEvent;
360            }
361        }
362    }
363
364    fn as_trap_frame(&self) -> TrapFrame {
365        TrapFrame {
366            rax: self.user_context.general.rax,
367            rbx: self.user_context.general.rbx,
368            rcx: self.user_context.general.rcx,
369            rdx: self.user_context.general.rdx,
370            rsi: self.user_context.general.rsi,
371            rdi: self.user_context.general.rdi,
372            rbp: self.user_context.general.rbp,
373            r8: self.user_context.general.r8,
374            r9: self.user_context.general.r9,
375            r10: self.user_context.general.r10,
376            r11: self.user_context.general.r11,
377            r12: self.user_context.general.r12,
378            r13: self.user_context.general.r13,
379            r14: self.user_context.general.r14,
380            r15: self.user_context.general.r15,
381            trap_num: self.user_context.trap_num,
382            error_code: self.user_context.error_code,
383            rip: self.user_context.general.rip,
384            cs: 0,
385            rflags: self.user_context.general.rflags,
386            rsp: self.user_context.general.rsp,
387            ss: 0,
388        }
389    }
390}
391
392/// As Osdev Wiki defines(<https://wiki.osdev.org/Exceptions>):
393/// CPU exceptions are classified as:
394///
395/// Faults: These can be corrected and the program may continue as if nothing happened.
396///
397/// Traps: Traps are reported immediately after the execution of the trapping instruction.
398///
399/// Aborts: Some severe unrecoverable error.
400///
401/// But there exists some vector which are special. Vector 1 can be both fault or trap and vector 2 is interrupt.
402/// So here we also define FaultOrTrap and Interrupt
403#[derive(Clone, Copy, Debug, Eq, PartialEq)]
404pub enum CpuExceptionType {
405    /// CPU faults. Faults can be corrected, and the program may continue as if nothing happened.
406    Fault,
407    /// CPU traps. Traps are reported immediately after the execution of the trapping instruction
408    Trap,
409    /// Faults or traps
410    FaultOrTrap,
411    /// CPU interrupts
412    Interrupt,
413    /// Some severe unrecoverable error
414    Abort,
415    /// Reserved for future use
416    Reserved,
417}
418
419impl CpuExceptionType {
420    /// Returns whether this exception type is a fault or a trap.
421    pub fn is_fault_or_trap(self) -> bool {
422        match self {
423            CpuExceptionType::Trap | CpuExceptionType::Fault | CpuExceptionType::FaultOrTrap => {
424                true
425            }
426            CpuExceptionType::Abort | CpuExceptionType::Interrupt | CpuExceptionType::Reserved => {
427                false
428            }
429        }
430    }
431}
432
433/// Architecture-specific data reported with a page-fault exception.
434#[derive(Clone, Copy, Debug, Eq, PartialEq)]
435pub struct RawPageFaultInfo {
436    /// The error code pushed by the CPU for this page fault.
437    pub error_code: PageFaultErrorCode,
438    /// The linear (virtual) address that triggered the fault (contents of CR2).
439    pub addr: Vaddr,
440}
441
442bitflags! {
443    /// Page Fault error code. Following the Intel Architectures Software Developer's Manual Volume 3
444    pub struct PageFaultErrorCode : usize{
445        /// 0 if no translation for the linear address.
446        const PRESENT       = 1 << 0;
447        /// 1 if the access was a write.
448        const WRITE         = 1 << 1;
449        /// 1 if the access was a user-mode access.
450        const USER          = 1 << 2;
451        /// 1 if there is no translation for the linear address
452        /// because a reserved bit was set.
453        const RESERVED      = 1 << 3;
454        /// 1 if the access was an instruction fetch.
455        const INSTRUCTION   = 1 << 4;
456        /// 1 if the access was a data access to a linear address with a protection key for which
457        /// the protection-key rights registers disallow access.
458        const PROTECTION    = 1 << 5;
459        /// 1 if the access was a shadow-stack access.
460        const SHADOW_STACK  = 1 << 6;
461        /// 1 if there is no translation for the linear address using HLAT paging.
462        const HLAT          = 1 << 7;
463        /// 1 if the exception is unrelated to paging and resulted from violation of SGX-specific
464        /// access-control requirements.
465        const SGX           = 1 << 15;
466    }
467}
468
469impl UserContextApi for UserContext {
470    fn set_instruction_pointer(&mut self, ip: usize) {
471        self.set_rip(ip);
472    }
473
474    fn set_stack_pointer(&mut self, sp: usize) {
475        self.set_rsp(sp)
476    }
477
478    fn stack_pointer(&self) -> usize {
479        self.rsp()
480    }
481
482    fn instruction_pointer(&self) -> usize {
483        self.rip()
484    }
485}
486
487macro_rules! cpu_context_impl_getter_setter {
488    ( $( [ $field: ident, $setter_name: ident] ),*) => {
489        impl UserContext {
490            $(
491                #[doc = concat!("Gets the value of ", stringify!($field))]
492                #[inline(always)]
493                pub fn $field(&self) -> usize {
494                    self.user_context.general.$field
495                }
496
497                #[doc = concat!("Sets the value of ", stringify!(field))]
498                #[inline(always)]
499                pub fn $setter_name(&mut self, $field: usize) {
500                    self.user_context.general.$field = $field;
501                }
502            )*
503        }
504    };
505}
506
507cpu_context_impl_getter_setter!(
508    [rax, set_rax],
509    [rbx, set_rbx],
510    [rcx, set_rcx],
511    [rdx, set_rdx],
512    [rsi, set_rsi],
513    [rdi, set_rdi],
514    [rbp, set_rbp],
515    [rsp, set_rsp],
516    [r8, set_r8],
517    [r9, set_r9],
518    [r10, set_r10],
519    [r11, set_r11],
520    [r12, set_r12],
521    [r13, set_r13],
522    [r14, set_r14],
523    [r15, set_r15],
524    [rip, set_rip],
525    [rflags, set_rflags]
526);
527
528/// The FPU context of user task.
529///
530/// This could be used for saving both legacy and modern state format.
531#[derive(Debug)]
532pub struct FpuContext {
533    xsave_area: Box<XSaveArea>,
534    area_size: usize,
535}
536
537impl FpuContext {
538    /// Creates a new FPU context.
539    pub fn new() -> Self {
540        let mut area_size = size_of::<FxSaveArea>();
541        if let Some(xsave_area_size) = XSAVE_AREA_SIZE.get() {
542            area_size = area_size.max(*xsave_area_size);
543        }
544
545        Self {
546            xsave_area: Box::new(XSaveArea::new()),
547            area_size,
548        }
549    }
550
551    /// Saves CPU's current FPU context to this instance.
552    pub fn save(&mut self) {
553        let mem_addr = self.as_bytes_mut().as_mut_ptr();
554
555        if XSTATE_MAX_FEATURES.is_completed() {
556            unsafe { _xsave64(mem_addr, XFEATURE_MASK_USER_RESTORE) };
557        } else {
558            unsafe { _fxsave64(mem_addr) };
559        }
560
561        debug!("Save FPU context");
562    }
563
564    /// Loads CPU's FPU context from this instance.
565    pub fn load(&self) {
566        let mem_addr = self.as_bytes().as_ptr();
567
568        if let Some(xstate_max_features) = XSTATE_MAX_FEATURES.get() {
569            let rs_mask = XFEATURE_MASK_USER_RESTORE & *xstate_max_features;
570
571            unsafe { _xrstor64(mem_addr, rs_mask) };
572        } else {
573            unsafe { _fxrstor64(mem_addr) };
574        }
575
576        debug!("Load FPU context");
577    }
578
579    /// Returns the FPU context as a byte slice.
580    pub fn as_bytes(&self) -> &[u8] {
581        &self.xsave_area.as_bytes()[..self.area_size]
582    }
583
584    /// Returns the FPU context as a mutable byte slice.
585    pub fn as_bytes_mut(&mut self) -> &mut [u8] {
586        &mut self.xsave_area.as_mut_bytes()[..self.area_size]
587    }
588}
589
590impl Default for FpuContext {
591    fn default() -> Self {
592        Self::new()
593    }
594}
595
596impl Clone for FpuContext {
597    fn clone(&self) -> Self {
598        let mut xsave_area = Box::new(XSaveArea::new());
599        xsave_area.fxsave_area = self.xsave_area.fxsave_area;
600        xsave_area.features = self.xsave_area.features;
601        xsave_area.compaction = self.xsave_area.compaction;
602        if self.area_size > size_of::<FxSaveArea>() {
603            let len = self.area_size - size_of::<FxSaveArea>() - 64;
604            xsave_area.extended_state_area[..len]
605                .copy_from_slice(&self.xsave_area.extended_state_area[..len]);
606        }
607
608        Self {
609            xsave_area,
610            area_size: self.area_size,
611        }
612    }
613}
614
615/// The modern FPU context format (as saved and restored by the `XSAVE` and `XRSTOR` instructions).
616#[repr(C)]
617#[repr(align(64))]
618#[derive(Clone, Copy, Debug, Pod)]
619struct XSaveArea {
620    fxsave_area: FxSaveArea,
621    features: u64,
622    compaction: u64,
623    reserved: [u64; 6],
624    extended_state_area: [u8; MAX_XSAVE_AREA_SIZE - size_of::<FxSaveArea>() - 64],
625}
626
627impl XSaveArea {
628    fn new() -> Self {
629        let features = if let Some(xstate_max_features) = XSTATE_MAX_FEATURES.get() {
630            XCr0::read().bits() & *xstate_max_features
631        } else {
632            0
633        };
634
635        let mut xsave_area = Self::new_zeroed();
636        // Set the initial values for the FPU context. Refer to Intel SDM, Table 10-1:
637        // "IA-32 and Intel® 64 Processor States Following Power-up, Reset, or INIT (Contd.)".
638        xsave_area.fxsave_area.control = 0x037F;
639        // Refer to Intel SDM, Volume 1, Section "x87 State". In the FXSAVE/XSAVE image the
640        // `tag` field contains an abridged x87 Tag Word (FTW) - a compact (8-bit) encoding
641        // used in saved/restore images. In this format, a bit value of 0 indicates the
642        // corresponding x87 register is empty (this is the inverse of the legacy 16-bit
643        // tag-word semantics). The `fninit` instruction clears all x87 registers, so the
644        // abridged tag must be initialized to 0 to represent an empty register set.
645        xsave_area.fxsave_area.tag = 0;
646        xsave_area.fxsave_area.mxcsr = 0x1F80;
647        xsave_area.features = features;
648
649        xsave_area
650    }
651}
652
653/// The legacy SSE/MMX FPU context format (as saved and restored by the `FXSAVE` and `FXRSTOR` instructions).
654#[repr(C)]
655#[repr(align(16))]
656#[derive(Clone, Copy, Debug, Pod)]
657struct FxSaveArea {
658    control: u16,         // x87 FPU Control Word
659    status: u16,          // x87 FPU Status Word
660    tag: u8,              // x87 FPU Tag Byte (abridged format)
661    reserved1: u8,        // Reserved
662    op: u16,              // x87 FPU Last Instruction Opcode
663    ip: u32,              // x87 FPU Instruction Pointer Offset
664    cs: u32,              // x87 FPU Instruction Pointer Selector
665    dp: u32,              // x87 FPU Instruction Operand (Data) Pointer Offset
666    ds: u32,              // x87 FPU Instruction Operand (Data) Pointer Selector
667    mxcsr: u32,           // MXCSR Register State
668    mxcsr_mask: u32,      // MXCSR Mask
669    st_space: [u32; 32], // x87 FPU or MMX technology registers (ST0-ST7 or MM0-MM7, 128 bits per field)
670    xmm_space: [u32; 64], // XMM registers (XMM0-XMM15, 128 bits per field)
671    reserved2: [u32; 12], // Reserved
672    reserved3: [u32; 12], // Software reserved
673}
674
675/// The XSTATE features (user & supervisor) supported by the processor.
676static XSTATE_MAX_FEATURES: Once<u64> = Once::new();
677
678/// Mask features which are restored when returning to user space.
679///
680/// X87 | SSE | AVX | OPMASK | ZMM_HI256 | HI16_ZMM
681const XFEATURE_MASK_USER_RESTORE: u64 = 0b1110_0111;
682
683/// The real size in bytes of the XSAVE area containing all states enabled by XCRO | IA32_XSS.
684static XSAVE_AREA_SIZE: Once<usize> = Once::new();
685
686/// The max size in bytes of the XSAVE area.
687const MAX_XSAVE_AREA_SIZE: usize = 4096;
688
689pub(in crate::arch) fn enable_essential_features() {
690    use super::extension::{IsaExtensions, has_extensions};
691
692    if has_extensions(IsaExtensions::XSAVE) {
693        XSTATE_MAX_FEATURES.call_once(|| super::cpuid::query_xstate_max_features().unwrap());
694        XSAVE_AREA_SIZE.call_once(|| {
695            let xsave_area_size = super::cpuid::query_xsave_area_size().unwrap() as usize;
696            assert!(xsave_area_size <= MAX_XSAVE_AREA_SIZE);
697            xsave_area_size
698        });
699    }
700
701    // We now assume that all x86-64 CPUs should have the FPU. Otherwise, we should check
702    // `has_extensions(IsaExtensions::FPU)` here.
703    {
704        let mut cr0 = Cr0::read();
705        cr0.remove(Cr0Flags::TASK_SWITCHED | Cr0Flags::EMULATE_COPROCESSOR);
706
707        unsafe {
708            Cr0::write(cr0);
709            // Flush out any pending x87 state.
710            core::arch::asm!("fninit");
711        }
712    }
713}
ostd/arch/x86/cpu/context/mod.rs

ostd/arch/x86/cpu/context/
mod.rs
