ostd/arch/x86/cpu/context/
mod.rs

1// SPDX-License-Identifier: MPL-2.0
2
3//! CPU execution context control.
4
5use alloc::boxed::Box;
6use core::arch::x86_64::{_fxrstor64, _fxsave64, _xrstor64, _xsave64};
7
8use bitflags::bitflags;
9use ostd_pod::{FromZeros, IntoBytes};
10use spin::Once;
11use x86::bits64::segmentation::{rdfsbase, rdgsbase, swapgs, wrfsbase, wrgsbase};
12use x86_64::registers::{
13    control::{Cr0, Cr0Flags},
14    rflags::RFlags,
15    xcontrol::XCr0,
16};
17
18use crate::{
19    arch::{
20        irq::HwIrqLine,
21        trap::{RawUserContext, TrapFrame},
22    },
23    cpu::PrivilegeLevel,
24    debug,
25    irq::{DisabledLocalIrqGuard, call_irq_callback_functions},
26    mm::Vaddr,
27    user::{ReturnReason, UserContextApi, UserContextApiInternal, UserModeHooks},
28};
29
30cfg_select! {
31    feature = "cvm_guest" => {
32        mod tdx;
33
34        use tdx::VirtualizationExceptionHandler;
35    }
36}
37
38/// RFLAGS bits that non-privileged code may modify.
39///
40/// This is the union of the flags that Linux allows `ptrace` and `rt_sigreturn`
41/// to modify. Because the set allowed by `rt_sigreturn` is a strict subset of
42/// the `ptrace` set (it excludes `NT`), this is effectively the set of flags
43/// that `ptrace` may modify.
44/// Reference: <https://elixir.bootlin.com/linux/v7.1/source/arch/x86/include/asm/sighandling.h#L11-L14>.
45/// Reference: <https://elixir.bootlin.com/linux/v7.1/source/arch/x86/kernel/ptrace.c#L158-L163>.
46pub const USER_MODIFIABLE_RFLAGS: usize = (RFlags::CARRY_FLAG.bits()
47    | RFlags::PARITY_FLAG.bits()
48    | RFlags::AUXILIARY_CARRY_FLAG.bits()
49    | RFlags::ZERO_FLAG.bits()
50    | RFlags::SIGN_FLAG.bits()
51    | RFlags::TRAP_FLAG.bits()
52    | RFlags::DIRECTION_FLAG.bits()
53    | RFlags::OVERFLOW_FLAG.bits()
54    | RFlags::RESUME_FLAG.bits()
55    | RFlags::ALIGNMENT_CHECK.bits()
56    | RFlags::NESTED_TASK.bits()) as usize;
57
58/// Userspace CPU context, including general-purpose registers and exception information.
59#[repr(C)]
60#[derive(Clone, Debug)]
61pub struct UserContext {
62    user_context: RawUserContext,
63    exception: Option<CpuException>,
64}
65
66impl Default for UserContext {
67    fn default() -> Self {
68        // RFLAGS bit 1 is reserved, and Linux always sets it to 1 on x86.
69        // Set the interrupt flag to enable the reception of external interrupts in user mode.
70        // Set the ID flag to indicate that the CPU supports the CPUID instruction.
71        const USER_MODE_INITIAL_RFLAGS: usize =
72            (1 << 1) | (RFlags::INTERRUPT_FLAG.bits() | RFlags::ID.bits()) as usize;
73
74        Self {
75            user_context: RawUserContext {
76                general: GeneralRegs::default(),
77                rflags: USER_MODE_INITIAL_RFLAGS,
78                trap_num: 0,
79                error_code: 0,
80            },
81            exception: None,
82        }
83    }
84}
85
86/// General registers.
87#[expect(missing_docs)]
88#[repr(C)]
89#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
90pub struct GeneralRegs {
91    pub rax: usize,
92    pub rbx: usize,
93    pub rcx: usize,
94    pub rdx: usize,
95    pub rsi: usize,
96    pub rdi: usize,
97    pub rbp: usize,
98    pub rsp: usize,
99    pub r8: usize,
100    pub r9: usize,
101    pub r10: usize,
102    pub r11: usize,
103    pub r12: usize,
104    pub r13: usize,
105    pub r14: usize,
106    pub r15: usize,
107    pub rip: usize,
108}
109
110/// The user-mode FS base register.
111#[derive(Clone, Copy, Debug, Default)]
112pub struct FsBase(usize);
113
114impl FsBase {
115    /// Creates a new `FsBase` with the given address.
116    pub fn new(addr: usize) -> Self {
117        Self(addr)
118    }
119
120    /// Returns the stored address.
121    pub fn addr(&self) -> usize {
122        self.0
123    }
124
125    /// Saves the current CPU FS base into this struct.
126    pub fn save(&mut self) {
127        // SAFETY: Reading the user FS base does not affect kernel code.
128        self.0 = unsafe { rdfsbase() as usize };
129    }
130
131    /// Loads this struct's FS base onto the CPU.
132    pub fn load(&self) {
133        // SAFETY: Writing the user FS base does not affect kernel code.
134        unsafe { wrfsbase(self.0 as u64) }
135    }
136}
137
138/// The user-mode GS base register.
139#[derive(Clone, Copy, Debug, Default)]
140pub struct GsBase(usize);
141
142impl GsBase {
143    /// Creates a new `GsBase` with the given address.
144    pub fn new(addr: usize) -> Self {
145        Self(addr)
146    }
147
148    /// Returns the stored address.
149    pub fn addr(&self) -> usize {
150        self.0
151    }
152
153    /// Saves the current CPU GS base into this struct.
154    pub fn save(&mut self, _guard: &DisabledLocalIrqGuard) {
155        // SAFETY:
156        // 1. In these steps, we have disabled the IRQ and are not using the kernel GS base.
157        // 2. Reading the user GS base does not affect kernel code.
158        unsafe {
159            swapgs();
160            self.0 = rdgsbase() as usize;
161            swapgs();
162        }
163    }
164
165    /// Loads this struct's GS base onto the CPU.
166    pub fn load(&self, _guard: &DisabledLocalIrqGuard) {
167        // SAFETY:
168        // 1. In these steps, we have disabled the IRQ and are not using the kernel GS base.
169        // 2. Writing the user GS base does not affect kernel code.
170        unsafe {
171            swapgs();
172            wrgsbase(self.0 as u64);
173            swapgs();
174        }
175    }
176}
177
178/// Architectural CPU exceptions (x86-64 vectors 0-31).
179///
180/// For the authoritative specification of each vector, see the
181/// Intel® 64 and IA-32 Architectures Software Developer’s Manual,
182/// Volume 3 “System Programming Guide”, Chapter 6 “Interrupt and Exception
183/// Handling”, in particular Section 6.15 “Exception and Interrupt
184/// Reference”.
185///
186/// Every enum variant corresponds to one exception defined by the
187/// Intel/AMD architecture.
188/// Variants that naturally carry an error code (or other error information)
189/// expose it through their associated data fields.
190//
191// TODO: Some exceptions (like `AlignmentCheck`) also push an
192//       error code onto the stack, but that detail is not yet represented
193//       in this type definition.
194#[derive(Clone, Copy, Debug, Eq, PartialEq)]
195pub enum CpuException {
196    ///  0 – #DE  Divide-by-zero error.
197    DivisionError,
198    ///  1 – #DB  Debug.
199    Debug,
200    ///  2 – NMI  Non-maskable interrupt.
201    NonMaskableInterrupt,
202    ///  3 – #BP  Breakpoint (INT3).
203    BreakPoint,
204    ///  4 – #OF  Overflow.
205    Overflow,
206    ///  5 – #BR  Bound-range exceeded.
207    BoundRangeExceeded,
208    ///  6 – #UD  Invalid or undefined opcode.
209    InvalidOpcode,
210    ///  7 – #NM  Device not available (FPU/MMX/SSE disabled).
211    DeviceNotAvailable,
212    ///  8 – #DF  Double fault (always pushes an error code of 0).
213    DoubleFault,
214    ///  9 – Coprocessor segment overrun (reserved on modern CPUs).
215    CoprocessorSegmentOverrun,
216    /// 10 – #TS  Invalid TSS.
217    InvalidTss(SelectorErrorCode),
218    /// 11 – #NP  Segment not present.
219    SegmentNotPresent(SelectorErrorCode),
220    /// 12 – #SS  Stack-segment fault.
221    StackSegmentFault(SelectorErrorCode),
222    /// 13 – #GP  General protection fault
223    GeneralProtectionFault(Option<SelectorErrorCode>),
224    /// 14 – #PF  Page fault.
225    PageFault(RawPageFaultInfo),
226    // 15: Reserved
227    /// 16 – #MF  x87 floating-point exception.
228    X87FloatingPointException,
229    /// 17 – #AC  Alignment check.
230    AlignmentCheck,
231    /// 18 – #MC  Machine check.
232    MachineCheck,
233    /// 19 – #XM / #XF  SIMD/FPU floating-point exception.
234    SIMDFloatingPointException,
235    /// 20 – #VE  Virtualization exception.
236    VirtualizationException,
237    /// 21 – #CP  Control protection exception (CET).
238    ControlProtectionException,
239    // 22-27: Reserved
240    /// 28 – #HV  Hypervisor injection exception.
241    HypervisorInjectionException,
242    /// 29 – #VC  VMM communication exception (SEV-ES GHCB).
243    VMMCommunicationException,
244    /// 30 – #SX  Security exception.
245    SecurityException,
246    // 31: Reserved
247    /// Catch-all for reserved or undefined vector numbers.
248    Reserved,
249}
250
251impl CpuException {
252    pub(in crate::arch) fn new(trap_num: usize, error_code: usize) -> Option<Self> {
253        let exception = match trap_num {
254            0 => Self::DivisionError,
255            1 => Self::Debug,
256            2 => Self::NonMaskableInterrupt,
257            3 => Self::BreakPoint,
258            4 => Self::Overflow,
259            5 => Self::BoundRangeExceeded,
260            6 => Self::InvalidOpcode,
261            7 => Self::DeviceNotAvailable,
262            8 => {
263                // A double fault will always generate an error code with a value of zero.
264                debug_assert_eq!(error_code, 0);
265                Self::DoubleFault
266            }
267            9 => Self::CoprocessorSegmentOverrun,
268            10 => Self::InvalidTss(SelectorErrorCode(error_code)),
269            11 => Self::SegmentNotPresent(SelectorErrorCode(error_code)),
270            12 => Self::StackSegmentFault(SelectorErrorCode(error_code)),
271            13 => {
272                let error_code = if error_code == 0 {
273                    None
274                } else {
275                    Some(SelectorErrorCode(error_code))
276                };
277                Self::GeneralProtectionFault(error_code)
278            }
279            14 => {
280                let page_fault_addr = x86_64::registers::control::Cr2::read_raw() as usize;
281                Self::PageFault(RawPageFaultInfo {
282                    error_code: PageFaultErrorCode::from_bits(error_code).unwrap(),
283                    addr: page_fault_addr,
284                })
285            }
286            // Reserved 15
287            16 => Self::X87FloatingPointException,
288            17 => Self::AlignmentCheck,
289            18 => Self::MachineCheck,
290            19 => Self::SIMDFloatingPointException,
291            20 => Self::VirtualizationException,
292            21 => Self::ControlProtectionException,
293            // Reserved 22-27
294            28 => Self::HypervisorInjectionException,
295            29 => Self::VMMCommunicationException,
296            30 => Self::SecurityException,
297            // Reserved 31
298            15 | 22..=27 | 31 => Self::Reserved,
299            _ => return None,
300        };
301
302        Some(exception)
303    }
304
305    const fn type_(&self) -> CpuExceptionType {
306        match self {
307            Self::Debug => CpuExceptionType::FaultOrTrap,
308            Self::NonMaskableInterrupt => CpuExceptionType::Interrupt,
309            Self::BreakPoint | Self::Overflow => CpuExceptionType::Trap,
310            Self::DoubleFault | Self::MachineCheck => CpuExceptionType::Abort,
311            Self::Reserved => CpuExceptionType::Reserved,
312            _ => CpuExceptionType::Fault,
313        }
314    }
315
316    pub(in crate::arch) const fn is_cpu_exception(trap_num: usize) -> bool {
317        trap_num <= 31
318    }
319}
320
321/// Selector error code.
322///
323/// Reference: <https://wiki.osdev.org/Exceptions#Selector_Error_Code>.
324#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
325pub struct SelectorErrorCode(usize);
326
327impl UserContext {
328    // Methods shared across all architectures (i.e., general registers and exceptions).
329
330    /// Returns a reference to the general registers.
331    pub fn general_regs(&self) -> &GeneralRegs {
332        &self.user_context.general
333    }
334
335    /// Returns a mutable reference to the general registers.
336    pub fn general_regs_mut(&mut self) -> &mut GeneralRegs {
337        &mut self.user_context.general
338    }
339
340    /// Takes the CPU exception out.
341    pub fn take_exception(&mut self) -> Option<CpuException> {
342        self.exception.take()
343    }
344
345    // Architecture-specific methods.
346
347    /// Gets the value of the RFLAGS register.
348    pub fn rflags(&self) -> usize {
349        self.user_context.rflags
350    }
351
352    /// Sets the value of the RFLAGS register.
353    ///
354    /// We only allow the setting or clearing of the bits in [`USER_MODIFIABLE_RFLAGS`].
355    /// Any other bits will be ignored and will remain unchanged.
356    pub fn set_rflags(&mut self, rflags: usize) {
357        self.user_context.rflags = (self.user_context.rflags & !USER_MODIFIABLE_RFLAGS)
358            | (rflags & USER_MODIFIABLE_RFLAGS);
359    }
360}
361
362impl UserContextApiInternal for UserContext {
363    fn execute<T: UserModeHooks>(&mut self, hooks: &T) -> ReturnReason {
364        const SYSCALL_TRAPNUM: usize = 0x100;
365
366        // Return when it is syscall or cpu exception type is Fault or Trap.
367        loop {
368            crate::task::scheduler::might_preempt();
369
370            let guard = crate::irq::disable_local();
371            hooks.pre_user_run(&guard);
372            self.user_context.run(guard);
373
374            let exception =
375                CpuException::new(self.user_context.trap_num, self.user_context.error_code);
376            match exception {
377                #[cfg(feature = "cvm_guest")]
378                Some(CpuException::VirtualizationException) => {
379                    let ve_handler = VirtualizationExceptionHandler::new();
380                    // Check out the doc of `VirtualizationExceptionHandler::new` to
381                    // see why IRQs must enabled _after_ instantiating a `VirtualizationExceptionHandler`.
382                    crate::arch::irq::enable_local();
383                    ve_handler.handle(self);
384                }
385                Some(exception) if exception.type_().is_fault_or_trap() => {
386                    crate::arch::irq::enable_local();
387                    self.exception = Some(exception);
388                    return ReturnReason::UserException;
389                }
390                Some(exception) => {
391                    panic!(
392                        "Cannot handle user CPU exception: {:?}; trapframe: {:?}",
393                        exception,
394                        self.as_trap_frame()
395                    );
396                }
397                None if self.user_context.trap_num == SYSCALL_TRAPNUM => {
398                    crate::arch::irq::enable_local();
399                    return ReturnReason::UserSyscall;
400                }
401                None => {
402                    call_irq_callback_functions(
403                        &self.as_trap_frame(),
404                        &HwIrqLine::new(self.as_trap_frame().trap_num as u8),
405                        PrivilegeLevel::User,
406                    );
407                    crate::arch::irq::enable_local();
408                }
409            }
410
411            if hooks.has_kernel_event() {
412                break ReturnReason::KernelEvent;
413            }
414        }
415    }
416
417    fn as_trap_frame(&self) -> TrapFrame {
418        TrapFrame {
419            rax: self.user_context.general.rax,
420            rbx: self.user_context.general.rbx,
421            rcx: self.user_context.general.rcx,
422            rdx: self.user_context.general.rdx,
423            rsi: self.user_context.general.rsi,
424            rdi: self.user_context.general.rdi,
425            rbp: self.user_context.general.rbp,
426            r8: self.user_context.general.r8,
427            r9: self.user_context.general.r9,
428            r10: self.user_context.general.r10,
429            r11: self.user_context.general.r11,
430            r12: self.user_context.general.r12,
431            r13: self.user_context.general.r13,
432            r14: self.user_context.general.r14,
433            r15: self.user_context.general.r15,
434            trap_num: self.user_context.trap_num,
435            error_code: self.user_context.error_code,
436            rip: self.user_context.general.rip,
437            cs: 0,
438            rflags: self.user_context.rflags,
439            rsp: self.user_context.general.rsp,
440            ss: 0,
441        }
442    }
443}
444
445/// Types of CPU exceptions.
446///
447/// As defined by Intel, there are three types of x86-64 CPU exceptions:
448///  - **Faults** can be corrected and the program may continue as if nothing happened.
449///  - **Traps** are reported immediately after the execution of the trapping instruction.
450///  - **Aborts** represent some unrecoverable errors.
451///
452/// However, there are some special vectors. Vector 1 can be either a fault or a trap, and vector 2
453/// is an interrupt. Here, we also define `FaultOrTrap` and `Interrupt`.
454#[derive(Clone, Copy, Debug, Eq, PartialEq)]
455enum CpuExceptionType {
456    /// Faults. They can be corrected and the program may continue as if nothing happened.
457    Fault,
458    /// Traps. They are reported immediately after the execution of the trapping instruction.
459    Trap,
460    /// Faults or traps.
461    FaultOrTrap,
462    /// Interrupts.
463    Interrupt,
464    /// Aborts. They represent some unrecoverable errors.
465    Abort,
466    /// Reserved for future use.
467    Reserved,
468}
469
470impl CpuExceptionType {
471    /// Returns whether this exception type is a fault or a trap.
472    fn is_fault_or_trap(self) -> bool {
473        match self {
474            CpuExceptionType::Trap | CpuExceptionType::Fault | CpuExceptionType::FaultOrTrap => {
475                true
476            }
477            CpuExceptionType::Abort | CpuExceptionType::Interrupt | CpuExceptionType::Reserved => {
478                false
479            }
480        }
481    }
482}
483
484/// Architecture-specific data reported with a page-fault exception.
485#[derive(Clone, Copy, Debug, Eq, PartialEq)]
486pub struct RawPageFaultInfo {
487    /// The error code pushed by the CPU for this page fault.
488    pub error_code: PageFaultErrorCode,
489    /// The linear (virtual) address that triggered the fault (contents of CR2).
490    pub addr: Vaddr,
491}
492
493bitflags! {
494    /// Page Fault error code. Following the Intel Architectures Software Developer's Manual Volume 3
495    pub struct PageFaultErrorCode : usize{
496        /// 0 if no translation for the linear address.
497        const PRESENT       = 1 << 0;
498        /// 1 if the access was a write.
499        const WRITE         = 1 << 1;
500        /// 1 if the access was a user-mode access.
501        const USER          = 1 << 2;
502        /// 1 if there is no translation for the linear address
503        /// because a reserved bit was set.
504        const RESERVED      = 1 << 3;
505        /// 1 if the access was an instruction fetch.
506        const INSTRUCTION   = 1 << 4;
507        /// 1 if the access was a data access to a linear address with a protection key for which
508        /// the protection-key rights registers disallow access.
509        const PROTECTION    = 1 << 5;
510        /// 1 if the access was a shadow-stack access.
511        const SHADOW_STACK  = 1 << 6;
512        /// 1 if there is no translation for the linear address using HLAT paging.
513        const HLAT          = 1 << 7;
514        /// 1 if the exception is unrelated to paging and resulted from violation of SGX-specific
515        /// access-control requirements.
516        const SGX           = 1 << 15;
517    }
518}
519
520impl UserContextApi for UserContext {
521    fn set_instruction_pointer(&mut self, ip: usize) {
522        self.set_rip(ip);
523    }
524
525    fn set_stack_pointer(&mut self, sp: usize) {
526        self.set_rsp(sp)
527    }
528
529    fn stack_pointer(&self) -> usize {
530        self.rsp()
531    }
532
533    fn instruction_pointer(&self) -> usize {
534        self.rip()
535    }
536}
537
538macro_rules! cpu_context_impl_getter_setter {
539    ( $( [ $field: ident, $setter_name: ident] ),*) => {
540        impl UserContext {
541            $(
542                #[doc = concat!("Gets the value of ", stringify!($field))]
543                #[inline(always)]
544                pub fn $field(&self) -> usize {
545                    self.user_context.general.$field
546                }
547
548                #[doc = concat!("Sets the value of ", stringify!(field))]
549                #[inline(always)]
550                pub fn $setter_name(&mut self, $field: usize) {
551                    self.user_context.general.$field = $field;
552                }
553            )*
554        }
555    };
556}
557
558cpu_context_impl_getter_setter!(
559    [rax, set_rax],
560    [rbx, set_rbx],
561    [rcx, set_rcx],
562    [rdx, set_rdx],
563    [rsi, set_rsi],
564    [rdi, set_rdi],
565    [rbp, set_rbp],
566    [rsp, set_rsp],
567    [r8, set_r8],
568    [r9, set_r9],
569    [r10, set_r10],
570    [r11, set_r11],
571    [r12, set_r12],
572    [r13, set_r13],
573    [r14, set_r14],
574    [r15, set_r15],
575    [rip, set_rip]
576);
577
578/// The FPU context of user task.
579///
580/// This could be used for saving both legacy and modern state format.
581#[derive(Debug)]
582pub struct FpuContext {
583    xsave_area: Box<XSaveArea>,
584    area_size: usize,
585}
586
587impl FpuContext {
588    /// Creates a new FPU context.
589    pub fn new() -> Self {
590        let mut area_size = size_of::<FxSaveArea>();
591        if let Some(xsave_area_size) = XSAVE_AREA_SIZE.get() {
592            area_size = area_size.max(*xsave_area_size);
593        }
594
595        Self {
596            xsave_area: Box::new(XSaveArea::new()),
597            area_size,
598        }
599    }
600
601    /// Saves CPU's current FPU context to this instance.
602    pub fn save(&mut self) {
603        let mem_addr = self.as_bytes_mut().as_mut_ptr();
604
605        if XSTATE_MAX_FEATURES.is_completed() {
606            unsafe { _xsave64(mem_addr, XFEATURE_MASK_USER_RESTORE) };
607        } else {
608            unsafe { _fxsave64(mem_addr) };
609        }
610
611        debug!("Save FPU context");
612    }
613
614    /// Loads CPU's FPU context from this instance.
615    pub fn load(&self) {
616        let mem_addr = self.as_bytes().as_ptr();
617
618        if let Some(xstate_max_features) = XSTATE_MAX_FEATURES.get() {
619            let rs_mask = XFEATURE_MASK_USER_RESTORE & *xstate_max_features;
620
621            unsafe { _xrstor64(mem_addr, rs_mask) };
622        } else {
623            unsafe { _fxrstor64(mem_addr) };
624        }
625
626        debug!("Load FPU context");
627    }
628
629    /// Returns the FPU context as a byte slice.
630    pub fn as_bytes(&self) -> &[u8] {
631        &self.xsave_area.as_bytes()[..self.area_size]
632    }
633
634    /// Returns the FPU context as a mutable byte slice.
635    pub fn as_bytes_mut(&mut self) -> &mut [u8] {
636        &mut self.xsave_area.as_mut_bytes()[..self.area_size]
637    }
638}
639
640impl Default for FpuContext {
641    fn default() -> Self {
642        Self::new()
643    }
644}
645
646impl Clone for FpuContext {
647    fn clone(&self) -> Self {
648        let mut xsave_area = Box::new(XSaveArea::new());
649        xsave_area.fxsave_area = self.xsave_area.fxsave_area;
650        xsave_area.features = self.xsave_area.features;
651        xsave_area.compaction = self.xsave_area.compaction;
652        if self.area_size > size_of::<FxSaveArea>() {
653            let len = self.area_size - size_of::<FxSaveArea>() - 64;
654            xsave_area.extended_state_area[..len]
655                .copy_from_slice(&self.xsave_area.extended_state_area[..len]);
656        }
657
658        Self {
659            xsave_area,
660            area_size: self.area_size,
661        }
662    }
663}
664
665/// The modern FPU context format (as saved and restored by the `XSAVE` and `XRSTOR` instructions).
666#[repr(C)]
667#[repr(align(64))]
668#[derive(Clone, Copy, Debug, Pod)]
669struct XSaveArea {
670    fxsave_area: FxSaveArea,
671    features: u64,
672    compaction: u64,
673    reserved: [u64; 6],
674    extended_state_area: [u8; MAX_XSAVE_AREA_SIZE - size_of::<FxSaveArea>() - 64],
675}
676
677impl XSaveArea {
678    fn new() -> Self {
679        let features = if let Some(xstate_max_features) = XSTATE_MAX_FEATURES.get() {
680            XCr0::read().bits() & *xstate_max_features
681        } else {
682            0
683        };
684
685        let mut xsave_area = Self::new_zeroed();
686        // Set the initial values for the FPU context. Refer to Intel SDM, Table 10-1:
687        // "IA-32 and Intel® 64 Processor States Following Power-up, Reset, or INIT (Contd.)".
688        xsave_area.fxsave_area.control = 0x037F;
689        // Refer to Intel SDM, Volume 1, Section "x87 State". In the FXSAVE/XSAVE image the
690        // `tag` field contains an abridged x87 Tag Word (FTW) - a compact (8-bit) encoding
691        // used in saved/restore images. In this format, a bit value of 0 indicates the
692        // corresponding x87 register is empty (this is the inverse of the legacy 16-bit
693        // tag-word semantics). The `fninit` instruction clears all x87 registers, so the
694        // abridged tag must be initialized to 0 to represent an empty register set.
695        xsave_area.fxsave_area.tag = 0;
696        xsave_area.fxsave_area.mxcsr = 0x1F80;
697        xsave_area.features = features;
698
699        xsave_area
700    }
701}
702
703/// The legacy SSE/MMX FPU context format (as saved and restored by the `FXSAVE` and `FXRSTOR` instructions).
704#[repr(C)]
705#[repr(align(16))]
706#[derive(Clone, Copy, Debug, Pod)]
707struct FxSaveArea {
708    control: u16,         // x87 FPU Control Word
709    status: u16,          // x87 FPU Status Word
710    tag: u8,              // x87 FPU Tag Byte (abridged format)
711    reserved1: u8,        // Reserved
712    op: u16,              // x87 FPU Last Instruction Opcode
713    ip: u32,              // x87 FPU Instruction Pointer Offset
714    cs: u32,              // x87 FPU Instruction Pointer Selector
715    dp: u32,              // x87 FPU Instruction Operand (Data) Pointer Offset
716    ds: u32,              // x87 FPU Instruction Operand (Data) Pointer Selector
717    mxcsr: u32,           // MXCSR Register State
718    mxcsr_mask: u32,      // MXCSR Mask
719    st_space: [u32; 32], // x87 FPU or MMX technology registers (ST0-ST7 or MM0-MM7, 128 bits per field)
720    xmm_space: [u32; 64], // XMM registers (XMM0-XMM15, 128 bits per field)
721    reserved2: [u32; 12], // Reserved
722    reserved3: [u32; 12], // Software reserved
723}
724
725/// The XSTATE features (user & supervisor) supported by the processor.
726static XSTATE_MAX_FEATURES: Once<u64> = Once::new();
727
728/// Mask features which are restored when returning to user space.
729///
730/// X87 | SSE | AVX | OPMASK | ZMM_HI256 | HI16_ZMM
731const XFEATURE_MASK_USER_RESTORE: u64 = 0b1110_0111;
732
733/// The real size in bytes of the XSAVE area containing all states enabled by XCRO | IA32_XSS.
734static XSAVE_AREA_SIZE: Once<usize> = Once::new();
735
736/// The max size in bytes of the XSAVE area.
737const MAX_XSAVE_AREA_SIZE: usize = 4096;
738
739pub(in crate::arch) fn enable_essential_features() {
740    use super::extension::{IsaExtensions, has_extensions};
741
742    if has_extensions(IsaExtensions::XSAVE) {
743        XSTATE_MAX_FEATURES.call_once(|| super::cpuid::query_xstate_max_features().unwrap());
744        XSAVE_AREA_SIZE.call_once(|| {
745            let xsave_area_size = super::cpuid::query_xsave_area_size().unwrap() as usize;
746            assert!(xsave_area_size <= MAX_XSAVE_AREA_SIZE);
747            xsave_area_size
748        });
749    }
750
751    // We now assume that all x86-64 CPUs should have the FPU. Otherwise, we should check
752    // `has_extensions(IsaExtensions::FPU)` here.
753    {
754        let mut cr0 = Cr0::read();
755        cr0.remove(Cr0Flags::TASK_SWITCHED | Cr0Flags::EMULATE_COPROCESSOR);
756
757        unsafe {
758            Cr0::write(cr0);
759            // Flush out any pending x87 state.
760            core::arch::asm!("fninit");
761        }
762    }
763}
ostd/arch/x86/cpu/context/mod.rs

ostd/arch/x86/cpu/context/
mod.rs
