ostd/arch/x86/
mod.rs

1// SPDX-License-Identifier: MPL-2.0
2
3//! Platform-specific code for the x86 platform.
4
5pub(crate) mod boot;
6pub mod cpu;
7pub mod device;
8pub(crate) mod io;
9pub(crate) mod iommu;
10pub mod irq;
11pub mod kernel;
12pub(crate) mod mm;
13mod power;
14pub mod serial;
15pub(crate) mod task;
16mod timer;
17pub mod trap;
18pub(crate) mod vm;
19
20#[cfg(feature = "cvm_guest")]
21pub(crate) mod tdx_guest;
22
23#[cfg(feature = "cvm_guest")]
24pub(crate) fn init_cvm_guest() {
25    use ::tdx_guest::{
26        SeptVeError, disable_sept_ve, init_tdx, metadata, reduce_unnecessary_ve,
27        tdcall::{InitError, write_td_metadata},
28        tdvmcall::report_fatal_error_simple,
29    };
30    match init_tdx() {
31        Ok(td_info) => {
32            reduce_unnecessary_ve().unwrap();
33            match disable_sept_ve(td_info.attributes) {
34                Ok(_) => {}
35                Err(SeptVeError::Misconfiguration) => {
36                    crate::early_println!(
37                        "[kernel] Error: TD misconfiguration: \
38                        The SEPT_VE_DISABLE bit of the TD attributes must be set by VMM \
39                        when running in non-debug mode and FLEXIBLE_PENDING_VE is not enabled."
40                    );
41                    report_fatal_error_simple("TD misconfiguration: SEPT #VE has to be disabled");
42                }
43                Err(e) => {
44                    crate::early_println!("[kernel] Error: Unexpected TDX error: {:?}", e);
45                    report_fatal_error_simple(
46                        "Disabling SEPT #VE failed due to unexpected TDX error",
47                    );
48                }
49            }
50            // Enable notification for zero step attack detection.
51            write_td_metadata(metadata::NOTIFY_ENABLES, 1, 1).unwrap();
52
53            crate::early_println!(
54                "[kernel] Intel TDX initialized\n[kernel] td gpaw: {}, td attributes: {:?}",
55                td_info.gpaw,
56                td_info.attributes
57            );
58        }
59        Err(InitError::TdxGetVpInfoError(td_call_error)) => {
60            crate::early_println!(
61                "[kernel] Intel TDX not initialized, Failed to get TD info. TD call error: {:?}",
62                td_call_error
63            );
64            report_fatal_error_simple("Intel TDX not initialized, Failed to get TD info.");
65        }
66        // The machine has no TDX support.
67        Err(_) => {}
68    }
69}
70
71/// Architecture-specific initialization on the bootstrapping processor.
72///
73/// It should be called when the heap and frame allocators are available.
74///
75/// # Safety
76///
77/// 1. This function must be called only once in the boot context of the
78///    bootstrapping processor.
79/// 2. This function must be called after the kernel page table is activated on
80///    the bootstrapping processor.
81pub(crate) unsafe fn late_init_on_bsp() {
82    // SAFETY: This is only called once on this BSP in the boot context.
83    unsafe { trap::init_on_cpu() };
84
85    // SAFETY: The caller ensures that this function is only called once on BSP,
86    // after the kernel page table is activated.
87    let io_mem_builder = unsafe { io::construct_io_mem_allocator_builder() };
88
89    kernel::apic::init(&io_mem_builder).expect("APIC doesn't exist");
90    irq::chip::init(&io_mem_builder);
91    irq::ipi::init();
92
93    kernel::tsc::init_tsc_freq();
94    timer::init_on_bsp();
95
96    // SAFETY: We're on the BSP and we're ready to boot all APs.
97    unsafe { crate::boot::smp::boot_all_aps() };
98
99    if_tdx_enabled!({
100    } else {
101        match iommu::init(&io_mem_builder) {
102            Ok(_) => {}
103            Err(err) => crate::warn!("IOMMU initialization error: {:?}", err),
104        }
105    });
106
107    // SAFETY:
108    // 1. All the system device memory have been removed from the builder.
109    // 2. All the port I/O regions belonging to the system device are defined using the macros.
110    // 3. `MAX_IO_PORT` defined in `crate::arch::io` is the maximum value specified by x86-64.
111    unsafe { crate::io::init(io_mem_builder) };
112
113    kernel::acpi::init();
114    power::init();
115}
116
117/// Initializes application-processor-specific state.
118///
119/// # Safety
120///
121/// 1. This function must be called only once on each application processor.
122/// 2. This function must be called after the BSP's call to [`late_init_on_bsp`]
123///    and before any other architecture-specific code in this module is called
124///    on this AP.
125pub(crate) unsafe fn init_on_ap() {
126    timer::init_on_ap();
127}
128
129/// Returns the frequency of TSC. The unit is Hz.
130pub fn tsc_freq() -> u64 {
131    use core::sync::atomic::Ordering;
132
133    kernel::tsc::TSC_FREQ.load(Ordering::Acquire)
134}
135
136/// Reads the current value of the processor's time-stamp counter (TSC).
137pub fn read_tsc() -> u64 {
138    use core::arch::x86_64::_rdtsc;
139
140    // SAFETY: It is safe to read a time-related counter.
141    unsafe { _rdtsc() }
142}
143
144/// Reads a hardware generated 64-bit random value.
145///
146/// Returns `None` if no random value was generated.
147pub fn read_random() -> Option<u64> {
148    use core::arch::x86_64::_rdrand64_step;
149
150    use cpu::extension::{IsaExtensions, has_extensions};
151
152    if !has_extensions(IsaExtensions::RDRAND) {
153        return None;
154    }
155
156    // Recommendation from "Intel(R) Digital Random Number Generator (DRNG) Software
157    // Implementation Guide" - Section 5.2.1 and "Intel(R) 64 and IA-32 Architectures
158    // Software Developer's Manual" - Volume 1 - Section 7.3.17.1.
159    const RETRY_LIMIT: usize = 10;
160
161    for _ in 0..RETRY_LIMIT {
162        let mut val = 0;
163        let generated = unsafe { _rdrand64_step(&mut val) };
164        if generated == 1 {
165            return Some(val);
166        }
167    }
168    None
169}
170
171pub(crate) fn enable_cpu_features() {
172    use cpu::extension::{IsaExtensions, has_extensions};
173    use x86_64::registers::{
174        control::{Cr0Flags, Cr4Flags},
175        xcontrol::XCr0Flags,
176    };
177
178    cpu::extension::init();
179    vm::init();
180
181    let mut cr0 = x86_64::registers::control::Cr0::read();
182    cr0 |= Cr0Flags::WRITE_PROTECT;
183    // These FPU control bits should be set for new CPUs (e.g., all CPUs with 64-bit support) and
184    // modern OSes. See recommendation from "Intel(R) 64 and IA-32 Architectures Software
185    // Developer's Manual" - Volume 3 - Section 10.2.1, Configuring the x87 FPU Environment.
186    cr0 |= Cr0Flags::NUMERIC_ERROR | Cr0Flags::MONITOR_COPROCESSOR;
187    unsafe { x86_64::registers::control::Cr0::write(cr0) };
188
189    let mut cr4 = x86_64::registers::control::Cr4::read();
190    cr4 |= Cr4Flags::OSFXSR | Cr4Flags::OSXMMEXCPT_ENABLE | Cr4Flags::PAGE_GLOBAL;
191    if has_extensions(IsaExtensions::XSAVE) {
192        cr4 |= Cr4Flags::OSXSAVE;
193    }
194    // For now, we unconditionally require the `rdfsbase`, `wrfsbase`, `rdgsbase`, and `wrgsbase`
195    // instructions because they are used when switching contexts, getting the address of a
196    // CPU-local variable, e.t.c. Meanwhile, this is at a very early stage of the boot process, so
197    // we want to avoid failing immediately even if we cannot enable these instructions (though the
198    // kernel will certainly fail later when they are absent).
199    //
200    // Note that this also enables the userspace to control their own FS/GS bases, which requires
201    // the kernel to properly deal with the arbitrary base values set by the userspace program.
202    if has_extensions(IsaExtensions::FSGSBASE) {
203        cr4 |= Cr4Flags::FSGSBASE;
204    }
205    unsafe { x86_64::registers::control::Cr4::write(cr4) };
206
207    if has_extensions(IsaExtensions::XSAVE) {
208        let mut xcr0 = x86_64::registers::xcontrol::XCr0::read();
209        xcr0 |= XCr0Flags::SSE;
210        if has_extensions(IsaExtensions::AVX) {
211            xcr0 |= XCr0Flags::AVX;
212        }
213        if has_extensions(IsaExtensions::AVX512F) {
214            xcr0 |= XCr0Flags::OPMASK | XCr0Flags::ZMM_HI256 | XCr0Flags::HI16_ZMM;
215        }
216        unsafe { x86_64::registers::xcontrol::XCr0::write(xcr0) };
217    }
218
219    cpu::context::enable_essential_features();
220
221    mm::enable_essential_features();
222}
223
224/// Inserts a TDX-specific code block.
225///
226/// This macro conditionally executes a TDX-specific code block based on the following conditions:
227/// (1) The `cvm_guest` feature is enabled at compile time.
228/// (2) The TDX feature is detected at runtime via `::tdx_guest::tdx_is_enabled()`.
229///
230/// If both conditions are met, the `if_block` is executed. If an `else_block` is provided, it will be executed
231/// when either the `cvm_guest` feature is not enabled or the TDX feature is not detected at runtime.
232#[macro_export]
233macro_rules! if_tdx_enabled {
234    // Match when there is an else block
235    ($if_block:block else $else_block:block) => {{
236        #[cfg(feature = "cvm_guest")]
237        {
238            if ::tdx_guest::tdx_is_enabled() {
239                $if_block
240            } else {
241                $else_block
242            }
243        }
244        #[cfg(not(feature = "cvm_guest"))]
245        {
246            $else_block
247        }
248    }};
249    // Match when there is no else block
250    ($if_block:block) => {{
251        #[cfg(feature = "cvm_guest")]
252        {
253            if ::tdx_guest::tdx_is_enabled() {
254                $if_block
255            }
256        }
257    }};
258}
259
260pub use if_tdx_enabled;
ostd/arch/x86/mod.rs

ostd/arch/x86/
mod.rs
