ostd/arch/x86/cpu/context/
mod.rs

1// SPDX-License-Identifier: MPL-2.0
2
3//! CPU execution context control.
4
5use alloc::boxed::Box;
6use core::arch::x86_64::{_fxrstor64, _fxsave64, _xrstor64, _xsave64};
7
8use bitflags::bitflags;
9use ostd_pod::{FromZeros, IntoBytes};
10use spin::Once;
11use x86::bits64::segmentation::{rdfsbase, rdgsbase, swapgs, wrfsbase, wrgsbase};
12use x86_64::registers::{
13    control::{Cr0, Cr0Flags},
14    rflags::RFlags,
15    xcontrol::XCr0,
16};
17
18use crate::{
19    arch::{
20        irq::HwIrqLine,
21        trap::{RawUserContext, TrapFrame},
22    },
23    cpu::PrivilegeLevel,
24    debug,
25    irq::{DisabledLocalIrqGuard, call_irq_callback_functions},
26    mm::Vaddr,
27    user::{ReturnReason, UserContextApi, UserContextApiInternal, UserModeHooks},
28};
29
30cfg_select! {
31    feature = "cvm_guest" => {
32        mod tdx;
33
34        use tdx::VirtualizationExceptionHandler;
35    }
36}
37
38/// Userspace CPU context, including general-purpose registers and exception information.
39#[repr(C)]
40#[derive(Clone, Debug, Default)]
41pub struct UserContext {
42    user_context: RawUserContext,
43    exception: Option<CpuException>,
44}
45
46/// General registers.
47#[expect(missing_docs)]
48#[repr(C)]
49#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
50pub struct GeneralRegs {
51    pub rax: usize,
52    pub rbx: usize,
53    pub rcx: usize,
54    pub rdx: usize,
55    pub rsi: usize,
56    pub rdi: usize,
57    pub rbp: usize,
58    pub rsp: usize,
59    pub r8: usize,
60    pub r9: usize,
61    pub r10: usize,
62    pub r11: usize,
63    pub r12: usize,
64    pub r13: usize,
65    pub r14: usize,
66    pub r15: usize,
67    pub rip: usize,
68    pub rflags: usize,
69}
70
71/// The user-mode FS base register.
72#[derive(Clone, Copy, Debug, Default)]
73pub struct FsBase(usize);
74
75impl FsBase {
76    /// Creates a new `FsBase` with the given address.
77    pub fn new(addr: usize) -> Self {
78        Self(addr)
79    }
80
81    /// Returns the stored address.
82    pub fn addr(&self) -> usize {
83        self.0
84    }
85
86    /// Saves the current CPU FS base into this struct.
87    pub fn save(&mut self) {
88        // SAFETY: Reading the user FS base does not affect kernel code.
89        self.0 = unsafe { rdfsbase() as usize };
90    }
91
92    /// Loads this struct's FS base onto the CPU.
93    pub fn load(&self) {
94        // SAFETY: Writing the user FS base does not affect kernel code.
95        unsafe { wrfsbase(self.0 as u64) }
96    }
97}
98
99/// The user-mode GS base register.
100#[derive(Clone, Copy, Debug, Default)]
101pub struct GsBase(usize);
102
103impl GsBase {
104    /// Creates a new `GsBase` with the given address.
105    pub fn new(addr: usize) -> Self {
106        Self(addr)
107    }
108
109    /// Returns the stored address.
110    pub fn addr(&self) -> usize {
111        self.0
112    }
113
114    /// Saves the current CPU GS base into this struct.
115    pub fn save(&mut self, _guard: &DisabledLocalIrqGuard) {
116        // SAFETY:
117        // 1. In these steps, we have disabled the IRQ and are not using the kernel GS base.
118        // 2. Reading the user GS base does not affect kernel code.
119        unsafe {
120            swapgs();
121            self.0 = rdgsbase() as usize;
122            swapgs();
123        }
124    }
125
126    /// Loads this struct's GS base onto the CPU.
127    pub fn load(&self, _guard: &DisabledLocalIrqGuard) {
128        // SAFETY:
129        // 1. In these steps, we have disabled the IRQ and are not using the kernel GS base.
130        // 2. Writing the user GS base does not affect kernel code.
131        unsafe {
132            swapgs();
133            wrgsbase(self.0 as u64);
134            swapgs();
135        }
136    }
137}
138
139/// Architectural CPU exceptions (x86-64 vectors 0-31).
140///
141/// For the authoritative specification of each vector, see the
142/// Intel® 64 and IA-32 Architectures Software Developer’s Manual,
143/// Volume 3 “System Programming Guide”, Chapter 6 “Interrupt and Exception
144/// Handling”, in particular Section 6.15 “Exception and Interrupt
145/// Reference”.
146///
147/// Every enum variant corresponds to one exception defined by the
148/// Intel/AMD architecture.
149/// Variants that naturally carry an error code (or other error information)
150/// expose it through their associated data fields.
151//
152// TODO: Some exceptions (like `AlignmentCheck`) also push an
153//       error code onto the stack, but that detail is not yet represented
154//       in this type definition.
155#[derive(Clone, Copy, Debug, Eq, PartialEq)]
156pub enum CpuException {
157    ///  0 – #DE  Divide-by-zero error.
158    DivisionError,
159    ///  1 – #DB  Debug.
160    Debug,
161    ///  2 – NMI  Non-maskable interrupt.
162    NonMaskableInterrupt,
163    ///  3 – #BP  Breakpoint (INT3).
164    BreakPoint,
165    ///  4 – #OF  Overflow.
166    Overflow,
167    ///  5 – #BR  Bound-range exceeded.
168    BoundRangeExceeded,
169    ///  6 – #UD  Invalid or undefined opcode.
170    InvalidOpcode,
171    ///  7 – #NM  Device not available (FPU/MMX/SSE disabled).
172    DeviceNotAvailable,
173    ///  8 – #DF  Double fault (always pushes an error code of 0).
174    DoubleFault,
175    ///  9 – Coprocessor segment overrun (reserved on modern CPUs).
176    CoprocessorSegmentOverrun,
177    /// 10 – #TS  Invalid TSS.
178    InvalidTss(SelectorErrorCode),
179    /// 11 – #NP  Segment not present.
180    SegmentNotPresent(SelectorErrorCode),
181    /// 12 – #SS  Stack-segment fault.
182    StackSegmentFault(SelectorErrorCode),
183    /// 13 – #GP  General protection fault
184    GeneralProtectionFault(Option<SelectorErrorCode>),
185    /// 14 – #PF  Page fault.
186    PageFault(RawPageFaultInfo),
187    // 15: Reserved
188    /// 16 – #MF  x87 floating-point exception.
189    X87FloatingPointException,
190    /// 17 – #AC  Alignment check.
191    AlignmentCheck,
192    /// 18 – #MC  Machine check.
193    MachineCheck,
194    /// 19 – #XM / #XF  SIMD/FPU floating-point exception.
195    SIMDFloatingPointException,
196    /// 20 – #VE  Virtualization exception.
197    VirtualizationException,
198    /// 21 – #CP  Control protection exception (CET).
199    ControlProtectionException,
200    // 22-27: Reserved
201    /// 28 – #HV  Hypervisor injection exception.
202    HypervisorInjectionException,
203    /// 29 – #VC  VMM communication exception (SEV-ES GHCB).
204    VMMCommunicationException,
205    /// 30 – #SX  Security exception.
206    SecurityException,
207    // 31: Reserved
208    /// Catch-all for reserved or undefined vector numbers.
209    Reserved,
210}
211
212impl CpuException {
213    pub(in crate::arch) fn new(trap_num: usize, error_code: usize) -> Option<Self> {
214        let exception = match trap_num {
215            0 => Self::DivisionError,
216            1 => Self::Debug,
217            2 => Self::NonMaskableInterrupt,
218            3 => Self::BreakPoint,
219            4 => Self::Overflow,
220            5 => Self::BoundRangeExceeded,
221            6 => Self::InvalidOpcode,
222            7 => Self::DeviceNotAvailable,
223            8 => {
224                // A double fault will always generate an error code with a value of zero.
225                debug_assert_eq!(error_code, 0);
226                Self::DoubleFault
227            }
228            9 => Self::CoprocessorSegmentOverrun,
229            10 => Self::InvalidTss(SelectorErrorCode(error_code)),
230            11 => Self::SegmentNotPresent(SelectorErrorCode(error_code)),
231            12 => Self::StackSegmentFault(SelectorErrorCode(error_code)),
232            13 => {
233                let error_code = if error_code == 0 {
234                    None
235                } else {
236                    Some(SelectorErrorCode(error_code))
237                };
238                Self::GeneralProtectionFault(error_code)
239            }
240            14 => {
241                let page_fault_addr = x86_64::registers::control::Cr2::read_raw() as usize;
242                Self::PageFault(RawPageFaultInfo {
243                    error_code: PageFaultErrorCode::from_bits(error_code).unwrap(),
244                    addr: page_fault_addr,
245                })
246            }
247            // Reserved 15
248            16 => Self::X87FloatingPointException,
249            17 => Self::AlignmentCheck,
250            18 => Self::MachineCheck,
251            19 => Self::SIMDFloatingPointException,
252            20 => Self::VirtualizationException,
253            21 => Self::ControlProtectionException,
254            // Reserved 22-27
255            28 => Self::HypervisorInjectionException,
256            29 => Self::VMMCommunicationException,
257            30 => Self::SecurityException,
258            // Reserved 31
259            15 | 22..=27 | 31 => Self::Reserved,
260            _ => return None,
261        };
262
263        Some(exception)
264    }
265
266    const fn type_(&self) -> CpuExceptionType {
267        match self {
268            Self::Debug => CpuExceptionType::FaultOrTrap,
269            Self::NonMaskableInterrupt => CpuExceptionType::Interrupt,
270            Self::BreakPoint | Self::Overflow => CpuExceptionType::Trap,
271            Self::DoubleFault | Self::MachineCheck => CpuExceptionType::Abort,
272            Self::Reserved => CpuExceptionType::Reserved,
273            _ => CpuExceptionType::Fault,
274        }
275    }
276
277    pub(in crate::arch) const fn is_cpu_exception(trap_num: usize) -> bool {
278        trap_num <= 31
279    }
280}
281
282/// Selector error code.
283///
284/// Reference: <https://wiki.osdev.org/Exceptions#Selector_Error_Code>.
285#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
286pub struct SelectorErrorCode(usize);
287
288impl UserContext {
289    /// Returns a reference to the general registers.
290    pub fn general_regs(&self) -> &GeneralRegs {
291        &self.user_context.general
292    }
293
294    /// Returns a mutable reference to the general registers
295    pub fn general_regs_mut(&mut self) -> &mut GeneralRegs {
296        &mut self.user_context.general
297    }
298
299    /// Takes the CPU exception out.
300    pub fn take_exception(&mut self) -> Option<CpuException> {
301        self.exception.take()
302    }
303}
304
305impl UserContextApiInternal for UserContext {
306    fn execute<T: UserModeHooks>(&mut self, hooks: &T) -> ReturnReason {
307        // Set the interrupt flag to enable the reception of external interrupts in user mode.
308        // Set the ID flag to indicate that the CPU supports the CPUID instruction.
309        self.user_context.general.rflags |= (RFlags::INTERRUPT_FLAG | RFlags::ID).bits() as usize;
310
311        const SYSCALL_TRAPNUM: usize = 0x100;
312
313        // Return when it is syscall or cpu exception type is Fault or Trap.
314        loop {
315            crate::task::scheduler::might_preempt();
316
317            let guard = crate::irq::disable_local();
318            hooks.pre_user_run(&guard);
319            self.user_context.run(guard);
320
321            let exception =
322                CpuException::new(self.user_context.trap_num, self.user_context.error_code);
323            match exception {
324                #[cfg(feature = "cvm_guest")]
325                Some(CpuException::VirtualizationException) => {
326                    let ve_handler = VirtualizationExceptionHandler::new();
327                    // Check out the doc of `VirtualizationExceptionHandler::new` to
328                    // see why IRQs must enabled _after_ instantiating a `VirtualizationExceptionHandler`.
329                    crate::arch::irq::enable_local();
330                    ve_handler.handle(self);
331                }
332                Some(exception) if exception.type_().is_fault_or_trap() => {
333                    crate::arch::irq::enable_local();
334                    self.exception = Some(exception);
335                    return ReturnReason::UserException;
336                }
337                Some(exception) => {
338                    panic!(
339                        "Cannot handle user CPU exception: {:?}; trapframe: {:?}",
340                        exception,
341                        self.as_trap_frame()
342                    );
343                }
344                None if self.user_context.trap_num == SYSCALL_TRAPNUM => {
345                    crate::arch::irq::enable_local();
346                    return ReturnReason::UserSyscall;
347                }
348                None => {
349                    call_irq_callback_functions(
350                        &self.as_trap_frame(),
351                        &HwIrqLine::new(self.as_trap_frame().trap_num as u8),
352                        PrivilegeLevel::User,
353                    );
354                    crate::arch::irq::enable_local();
355                }
356            }
357
358            if hooks.has_kernel_event() {
359                break ReturnReason::KernelEvent;
360            }
361        }
362    }
363
364    fn as_trap_frame(&self) -> TrapFrame {
365        TrapFrame {
366            rax: self.user_context.general.rax,
367            rbx: self.user_context.general.rbx,
368            rcx: self.user_context.general.rcx,
369            rdx: self.user_context.general.rdx,
370            rsi: self.user_context.general.rsi,
371            rdi: self.user_context.general.rdi,
372            rbp: self.user_context.general.rbp,
373            r8: self.user_context.general.r8,
374            r9: self.user_context.general.r9,
375            r10: self.user_context.general.r10,
376            r11: self.user_context.general.r11,
377            r12: self.user_context.general.r12,
378            r13: self.user_context.general.r13,
379            r14: self.user_context.general.r14,
380            r15: self.user_context.general.r15,
381            trap_num: self.user_context.trap_num,
382            error_code: self.user_context.error_code,
383            rip: self.user_context.general.rip,
384            cs: 0,
385            rflags: self.user_context.general.rflags,
386            rsp: self.user_context.general.rsp,
387            ss: 0,
388        }
389    }
390}
391
392/// Types of CPU exceptions.
393///
394/// As defined by Intel, there are three types of x86-64 CPU exceptions:
395///  - **Faults** can be corrected and the program may continue as if nothing happened.
396///  - **Traps** are reported immediately after the execution of the trapping instruction.
397///  - **Aborts** represent some unrecoverable errors.
398///
399/// However, there are some special vectors. Vector 1 can be either a fault or a trap, and vector 2
400/// is an interrupt. Here, we also define `FaultOrTrap` and `Interrupt`.
401#[derive(Clone, Copy, Debug, Eq, PartialEq)]
402enum CpuExceptionType {
403    /// Faults. They can be corrected and the program may continue as if nothing happened.
404    Fault,
405    /// Traps. They are reported immediately after the execution of the trapping instruction.
406    Trap,
407    /// Faults or traps.
408    FaultOrTrap,
409    /// Interrupts.
410    Interrupt,
411    /// Aborts. They represent some unrecoverable errors.
412    Abort,
413    /// Reserved for future use.
414    Reserved,
415}
416
417impl CpuExceptionType {
418    /// Returns whether this exception type is a fault or a trap.
419    fn is_fault_or_trap(self) -> bool {
420        match self {
421            CpuExceptionType::Trap | CpuExceptionType::Fault | CpuExceptionType::FaultOrTrap => {
422                true
423            }
424            CpuExceptionType::Abort | CpuExceptionType::Interrupt | CpuExceptionType::Reserved => {
425                false
426            }
427        }
428    }
429}
430
431/// Architecture-specific data reported with a page-fault exception.
432#[derive(Clone, Copy, Debug, Eq, PartialEq)]
433pub struct RawPageFaultInfo {
434    /// The error code pushed by the CPU for this page fault.
435    pub error_code: PageFaultErrorCode,
436    /// The linear (virtual) address that triggered the fault (contents of CR2).
437    pub addr: Vaddr,
438}
439
440bitflags! {
441    /// Page Fault error code. Following the Intel Architectures Software Developer's Manual Volume 3
442    pub struct PageFaultErrorCode : usize{
443        /// 0 if no translation for the linear address.
444        const PRESENT       = 1 << 0;
445        /// 1 if the access was a write.
446        const WRITE         = 1 << 1;
447        /// 1 if the access was a user-mode access.
448        const USER          = 1 << 2;
449        /// 1 if there is no translation for the linear address
450        /// because a reserved bit was set.
451        const RESERVED      = 1 << 3;
452        /// 1 if the access was an instruction fetch.
453        const INSTRUCTION   = 1 << 4;
454        /// 1 if the access was a data access to a linear address with a protection key for which
455        /// the protection-key rights registers disallow access.
456        const PROTECTION    = 1 << 5;
457        /// 1 if the access was a shadow-stack access.
458        const SHADOW_STACK  = 1 << 6;
459        /// 1 if there is no translation for the linear address using HLAT paging.
460        const HLAT          = 1 << 7;
461        /// 1 if the exception is unrelated to paging and resulted from violation of SGX-specific
462        /// access-control requirements.
463        const SGX           = 1 << 15;
464    }
465}
466
467impl UserContextApi for UserContext {
468    fn set_instruction_pointer(&mut self, ip: usize) {
469        self.set_rip(ip);
470    }
471
472    fn set_stack_pointer(&mut self, sp: usize) {
473        self.set_rsp(sp)
474    }
475
476    fn stack_pointer(&self) -> usize {
477        self.rsp()
478    }
479
480    fn instruction_pointer(&self) -> usize {
481        self.rip()
482    }
483}
484
485macro_rules! cpu_context_impl_getter_setter {
486    ( $( [ $field: ident, $setter_name: ident] ),*) => {
487        impl UserContext {
488            $(
489                #[doc = concat!("Gets the value of ", stringify!($field))]
490                #[inline(always)]
491                pub fn $field(&self) -> usize {
492                    self.user_context.general.$field
493                }
494
495                #[doc = concat!("Sets the value of ", stringify!(field))]
496                #[inline(always)]
497                pub fn $setter_name(&mut self, $field: usize) {
498                    self.user_context.general.$field = $field;
499                }
500            )*
501        }
502    };
503}
504
505cpu_context_impl_getter_setter!(
506    [rax, set_rax],
507    [rbx, set_rbx],
508    [rcx, set_rcx],
509    [rdx, set_rdx],
510    [rsi, set_rsi],
511    [rdi, set_rdi],
512    [rbp, set_rbp],
513    [rsp, set_rsp],
514    [r8, set_r8],
515    [r9, set_r9],
516    [r10, set_r10],
517    [r11, set_r11],
518    [r12, set_r12],
519    [r13, set_r13],
520    [r14, set_r14],
521    [r15, set_r15],
522    [rip, set_rip],
523    [rflags, set_rflags]
524);
525
526/// The FPU context of user task.
527///
528/// This could be used for saving both legacy and modern state format.
529#[derive(Debug)]
530pub struct FpuContext {
531    xsave_area: Box<XSaveArea>,
532    area_size: usize,
533}
534
535impl FpuContext {
536    /// Creates a new FPU context.
537    pub fn new() -> Self {
538        let mut area_size = size_of::<FxSaveArea>();
539        if let Some(xsave_area_size) = XSAVE_AREA_SIZE.get() {
540            area_size = area_size.max(*xsave_area_size);
541        }
542
543        Self {
544            xsave_area: Box::new(XSaveArea::new()),
545            area_size,
546        }
547    }
548
549    /// Saves CPU's current FPU context to this instance.
550    pub fn save(&mut self) {
551        let mem_addr = self.as_bytes_mut().as_mut_ptr();
552
553        if XSTATE_MAX_FEATURES.is_completed() {
554            unsafe { _xsave64(mem_addr, XFEATURE_MASK_USER_RESTORE) };
555        } else {
556            unsafe { _fxsave64(mem_addr) };
557        }
558
559        debug!("Save FPU context");
560    }
561
562    /// Loads CPU's FPU context from this instance.
563    pub fn load(&self) {
564        let mem_addr = self.as_bytes().as_ptr();
565
566        if let Some(xstate_max_features) = XSTATE_MAX_FEATURES.get() {
567            let rs_mask = XFEATURE_MASK_USER_RESTORE & *xstate_max_features;
568
569            unsafe { _xrstor64(mem_addr, rs_mask) };
570        } else {
571            unsafe { _fxrstor64(mem_addr) };
572        }
573
574        debug!("Load FPU context");
575    }
576
577    /// Returns the FPU context as a byte slice.
578    pub fn as_bytes(&self) -> &[u8] {
579        &self.xsave_area.as_bytes()[..self.area_size]
580    }
581
582    /// Returns the FPU context as a mutable byte slice.
583    pub fn as_bytes_mut(&mut self) -> &mut [u8] {
584        &mut self.xsave_area.as_mut_bytes()[..self.area_size]
585    }
586}
587
588impl Default for FpuContext {
589    fn default() -> Self {
590        Self::new()
591    }
592}
593
594impl Clone for FpuContext {
595    fn clone(&self) -> Self {
596        let mut xsave_area = Box::new(XSaveArea::new());
597        xsave_area.fxsave_area = self.xsave_area.fxsave_area;
598        xsave_area.features = self.xsave_area.features;
599        xsave_area.compaction = self.xsave_area.compaction;
600        if self.area_size > size_of::<FxSaveArea>() {
601            let len = self.area_size - size_of::<FxSaveArea>() - 64;
602            xsave_area.extended_state_area[..len]
603                .copy_from_slice(&self.xsave_area.extended_state_area[..len]);
604        }
605
606        Self {
607            xsave_area,
608            area_size: self.area_size,
609        }
610    }
611}
612
613/// The modern FPU context format (as saved and restored by the `XSAVE` and `XRSTOR` instructions).
614#[repr(C)]
615#[repr(align(64))]
616#[derive(Clone, Copy, Debug, Pod)]
617struct XSaveArea {
618    fxsave_area: FxSaveArea,
619    features: u64,
620    compaction: u64,
621    reserved: [u64; 6],
622    extended_state_area: [u8; MAX_XSAVE_AREA_SIZE - size_of::<FxSaveArea>() - 64],
623}
624
625impl XSaveArea {
626    fn new() -> Self {
627        let features = if let Some(xstate_max_features) = XSTATE_MAX_FEATURES.get() {
628            XCr0::read().bits() & *xstate_max_features
629        } else {
630            0
631        };
632
633        let mut xsave_area = Self::new_zeroed();
634        // Set the initial values for the FPU context. Refer to Intel SDM, Table 10-1:
635        // "IA-32 and Intel® 64 Processor States Following Power-up, Reset, or INIT (Contd.)".
636        xsave_area.fxsave_area.control = 0x037F;
637        // Refer to Intel SDM, Volume 1, Section "x87 State". In the FXSAVE/XSAVE image the
638        // `tag` field contains an abridged x87 Tag Word (FTW) - a compact (8-bit) encoding
639        // used in saved/restore images. In this format, a bit value of 0 indicates the
640        // corresponding x87 register is empty (this is the inverse of the legacy 16-bit
641        // tag-word semantics). The `fninit` instruction clears all x87 registers, so the
642        // abridged tag must be initialized to 0 to represent an empty register set.
643        xsave_area.fxsave_area.tag = 0;
644        xsave_area.fxsave_area.mxcsr = 0x1F80;
645        xsave_area.features = features;
646
647        xsave_area
648    }
649}
650
651/// The legacy SSE/MMX FPU context format (as saved and restored by the `FXSAVE` and `FXRSTOR` instructions).
652#[repr(C)]
653#[repr(align(16))]
654#[derive(Clone, Copy, Debug, Pod)]
655struct FxSaveArea {
656    control: u16,         // x87 FPU Control Word
657    status: u16,          // x87 FPU Status Word
658    tag: u8,              // x87 FPU Tag Byte (abridged format)
659    reserved1: u8,        // Reserved
660    op: u16,              // x87 FPU Last Instruction Opcode
661    ip: u32,              // x87 FPU Instruction Pointer Offset
662    cs: u32,              // x87 FPU Instruction Pointer Selector
663    dp: u32,              // x87 FPU Instruction Operand (Data) Pointer Offset
664    ds: u32,              // x87 FPU Instruction Operand (Data) Pointer Selector
665    mxcsr: u32,           // MXCSR Register State
666    mxcsr_mask: u32,      // MXCSR Mask
667    st_space: [u32; 32], // x87 FPU or MMX technology registers (ST0-ST7 or MM0-MM7, 128 bits per field)
668    xmm_space: [u32; 64], // XMM registers (XMM0-XMM15, 128 bits per field)
669    reserved2: [u32; 12], // Reserved
670    reserved3: [u32; 12], // Software reserved
671}
672
673/// The XSTATE features (user & supervisor) supported by the processor.
674static XSTATE_MAX_FEATURES: Once<u64> = Once::new();
675
676/// Mask features which are restored when returning to user space.
677///
678/// X87 | SSE | AVX | OPMASK | ZMM_HI256 | HI16_ZMM
679const XFEATURE_MASK_USER_RESTORE: u64 = 0b1110_0111;
680
681/// The real size in bytes of the XSAVE area containing all states enabled by XCRO | IA32_XSS.
682static XSAVE_AREA_SIZE: Once<usize> = Once::new();
683
684/// The max size in bytes of the XSAVE area.
685const MAX_XSAVE_AREA_SIZE: usize = 4096;
686
687pub(in crate::arch) fn enable_essential_features() {
688    use super::extension::{IsaExtensions, has_extensions};
689
690    if has_extensions(IsaExtensions::XSAVE) {
691        XSTATE_MAX_FEATURES.call_once(|| super::cpuid::query_xstate_max_features().unwrap());
692        XSAVE_AREA_SIZE.call_once(|| {
693            let xsave_area_size = super::cpuid::query_xsave_area_size().unwrap() as usize;
694            assert!(xsave_area_size <= MAX_XSAVE_AREA_SIZE);
695            xsave_area_size
696        });
697    }
698
699    // We now assume that all x86-64 CPUs should have the FPU. Otherwise, we should check
700    // `has_extensions(IsaExtensions::FPU)` here.
701    {
702        let mut cr0 = Cr0::read();
703        cr0.remove(Cr0Flags::TASK_SWITCHED | Cr0Flags::EMULATE_COPROCESSOR);
704
705        unsafe {
706            Cr0::write(cr0);
707            // Flush out any pending x87 state.
708            core::arch::asm!("fninit");
709        }
710    }
711}
ostd/arch/x86/cpu/context/mod.rs

ostd/arch/x86/cpu/context/
mod.rs
