ostd/arch/x86/
mod.rs

1// SPDX-License-Identifier: MPL-2.0
2
3//! Platform-specific code for the x86 platform.
4
5pub(crate) mod boot;
6pub mod cpu;
7pub mod device;
8pub(crate) mod io;
9pub(crate) mod iommu;
10pub mod irq;
11pub mod kernel;
12pub(crate) mod mm;
13mod power;
14pub mod serial;
15pub(crate) mod task;
16mod timer;
17pub mod trap;
18
19#[cfg(feature = "cvm_guest")]
20pub(crate) mod tdx_guest;
21
22#[cfg(feature = "cvm_guest")]
23pub(crate) fn init_cvm_guest() {
24    use ::tdx_guest::{
25        SeptVeError, disable_sept_ve, init_tdx, metadata, reduce_unnecessary_ve,
26        tdcall::{InitError, write_td_metadata},
27        tdvmcall::report_fatal_error_simple,
28    };
29    match init_tdx() {
30        Ok(td_info) => {
31            reduce_unnecessary_ve().unwrap();
32            match disable_sept_ve(td_info.attributes) {
33                Ok(_) => {}
34                Err(SeptVeError::Misconfiguration) => {
35                    crate::early_println!(
36                        "[kernel] Error: TD misconfiguration: \
37                        The SEPT_VE_DISABLE bit of the TD attributes must be set by VMM \
38                        when running in non-debug mode and FLEXIBLE_PENDING_VE is not enabled."
39                    );
40                    report_fatal_error_simple("TD misconfiguration: SEPT #VE has to be disabled");
41                }
42                Err(e) => {
43                    crate::early_println!("[kernel] Error: Unexpected TDX error: {:?}", e);
44                    report_fatal_error_simple(
45                        "Disabling SEPT #VE failed due to unexpected TDX error",
46                    );
47                }
48            }
49            // Enable notification for zero step attack detection.
50            write_td_metadata(metadata::NOTIFY_ENABLES, 1, 1).unwrap();
51
52            crate::early_println!(
53                "[kernel] Intel TDX initialized\n[kernel] td gpaw: {}, td attributes: {:?}",
54                td_info.gpaw,
55                td_info.attributes
56            );
57        }
58        Err(InitError::TdxGetVpInfoError(td_call_error)) => {
59            crate::early_println!(
60                "[kernel] Intel TDX not initialized, Failed to get TD info. TD call error: {:?}",
61                td_call_error
62            );
63            report_fatal_error_simple("Intel TDX not initialized, Failed to get TD info.");
64        }
65        // The machine has no TDX support.
66        Err(_) => {}
67    }
68}
69
70/// Architecture-specific initialization on the bootstrapping processor.
71///
72/// It should be called when the heap and frame allocators are available.
73///
74/// # Safety
75///
76/// 1. This function must be called only once in the boot context of the
77///    bootstrapping processor.
78/// 2. This function must be called after the kernel page table is activated on
79///    the bootstrapping processor.
80pub(crate) unsafe fn late_init_on_bsp() {
81    // SAFETY: This is only called once on this BSP in the boot context.
82    unsafe { trap::init_on_cpu() };
83
84    // SAFETY: The caller ensures that this function is only called once on BSP,
85    // after the kernel page table is activated.
86    let io_mem_builder = unsafe { io::construct_io_mem_allocator_builder() };
87
88    kernel::apic::init(&io_mem_builder).expect("APIC doesn't exist");
89    irq::chip::init(&io_mem_builder);
90    irq::ipi::init();
91
92    kernel::tsc::init_tsc_freq();
93    timer::init_on_bsp();
94
95    // SAFETY: We're on the BSP and we're ready to boot all APs.
96    unsafe { crate::boot::smp::boot_all_aps() };
97
98    if_tdx_enabled!({
99    } else {
100        match iommu::init(&io_mem_builder) {
101            Ok(_) => {}
102            Err(err) => log::warn!("IOMMU initialization error:{:?}", err),
103        }
104    });
105
106    // SAFETY:
107    // 1. All the system device memory have been removed from the builder.
108    // 2. All the port I/O regions belonging to the system device are defined using the macros.
109    // 3. `MAX_IO_PORT` defined in `crate::arch::io` is the maximum value specified by x86-64.
110    unsafe { crate::io::init(io_mem_builder) };
111
112    kernel::acpi::init();
113    power::init();
114}
115
116/// Initializes application-processor-specific state.
117///
118/// # Safety
119///
120/// 1. This function must be called only once on each application processor.
121/// 2. This function must be called after the BSP's call to [`late_init_on_bsp`]
122///    and before any other architecture-specific code in this module is called
123///    on this AP.
124pub(crate) unsafe fn init_on_ap() {
125    timer::init_on_ap();
126}
127
128/// Returns the frequency of TSC. The unit is Hz.
129pub fn tsc_freq() -> u64 {
130    use core::sync::atomic::Ordering;
131
132    kernel::tsc::TSC_FREQ.load(Ordering::Acquire)
133}
134
135/// Reads the current value of the processor's time-stamp counter (TSC).
136pub fn read_tsc() -> u64 {
137    use core::arch::x86_64::_rdtsc;
138
139    // SAFETY: It is safe to read a time-related counter.
140    unsafe { _rdtsc() }
141}
142
143/// Reads a hardware generated 64-bit random value.
144///
145/// Returns `None` if no random value was generated.
146pub fn read_random() -> Option<u64> {
147    use core::arch::x86_64::_rdrand64_step;
148
149    use cpu::extension::{IsaExtensions, has_extensions};
150
151    if !has_extensions(IsaExtensions::RDRAND) {
152        return None;
153    }
154
155    // Recommendation from "Intel(R) Digital Random Number Generator (DRNG) Software
156    // Implementation Guide" - Section 5.2.1 and "Intel(R) 64 and IA-32 Architectures
157    // Software Developer's Manual" - Volume 1 - Section 7.3.17.1.
158    const RETRY_LIMIT: usize = 10;
159
160    for _ in 0..RETRY_LIMIT {
161        let mut val = 0;
162        let generated = unsafe { _rdrand64_step(&mut val) };
163        if generated == 1 {
164            return Some(val);
165        }
166    }
167    None
168}
169
170pub(crate) fn enable_cpu_features() {
171    use cpu::extension::{IsaExtensions, has_extensions};
172    use x86_64::registers::{
173        control::{Cr0Flags, Cr4Flags},
174        xcontrol::XCr0Flags,
175    };
176
177    cpu::extension::init();
178
179    let mut cr0 = x86_64::registers::control::Cr0::read();
180    cr0 |= Cr0Flags::WRITE_PROTECT;
181    // These FPU control bits should be set for new CPUs (e.g., all CPUs with 64-bit support) and
182    // modern OSes. See recommendation from "Intel(R) 64 and IA-32 Architectures Software
183    // Developer's Manual" - Volume 3 - Section 10.2.1, Configuring the x87 FPU Environment.
184    cr0 |= Cr0Flags::NUMERIC_ERROR | Cr0Flags::MONITOR_COPROCESSOR;
185    unsafe { x86_64::registers::control::Cr0::write(cr0) };
186
187    let mut cr4 = x86_64::registers::control::Cr4::read();
188    cr4 |= Cr4Flags::OSFXSR | Cr4Flags::OSXMMEXCPT_ENABLE | Cr4Flags::PAGE_GLOBAL;
189    if has_extensions(IsaExtensions::XSAVE) {
190        cr4 |= Cr4Flags::OSXSAVE;
191    }
192    // For now, we unconditionally require the `rdfsbase`, `wrfsbase`, `rdgsbase`, and `wrgsbase`
193    // instructions because they are used when switching contexts, getting the address of a
194    // CPU-local variable, e.t.c. Meanwhile, this is at a very early stage of the boot process, so
195    // we want to avoid failing immediately even if we cannot enable these instructions (though the
196    // kernel will certainly fail later when they are absent).
197    //
198    // Note that this also enables the userspace to control their own FS/GS bases, which requires
199    // the kernel to properly deal with the arbitrary base values set by the userspace program.
200    if has_extensions(IsaExtensions::FSGSBASE) {
201        cr4 |= Cr4Flags::FSGSBASE;
202    }
203    unsafe { x86_64::registers::control::Cr4::write(cr4) };
204
205    if has_extensions(IsaExtensions::XSAVE) {
206        let mut xcr0 = x86_64::registers::xcontrol::XCr0::read();
207        xcr0 |= XCr0Flags::SSE;
208        if has_extensions(IsaExtensions::AVX) {
209            xcr0 |= XCr0Flags::AVX;
210        }
211        if has_extensions(IsaExtensions::AVX512F) {
212            xcr0 |= XCr0Flags::OPMASK | XCr0Flags::ZMM_HI256 | XCr0Flags::HI16_ZMM;
213        }
214        unsafe { x86_64::registers::xcontrol::XCr0::write(xcr0) };
215    }
216
217    cpu::context::enable_essential_features();
218
219    mm::enable_essential_features();
220}
221
222/// Inserts a TDX-specific code block.
223///
224/// This macro conditionally executes a TDX-specific code block based on the following conditions:
225/// (1) The `cvm_guest` feature is enabled at compile time.
226/// (2) The TDX feature is detected at runtime via `::tdx_guest::tdx_is_enabled()`.
227///
228/// If both conditions are met, the `if_block` is executed. If an `else_block` is provided, it will be executed
229/// when either the `cvm_guest` feature is not enabled or the TDX feature is not detected at runtime.
230#[macro_export]
231macro_rules! if_tdx_enabled {
232    // Match when there is an else block
233    ($if_block:block else $else_block:block) => {{
234        #[cfg(feature = "cvm_guest")]
235        {
236            if ::tdx_guest::tdx_is_enabled() {
237                $if_block
238            } else {
239                $else_block
240            }
241        }
242        #[cfg(not(feature = "cvm_guest"))]
243        {
244            $else_block
245        }
246    }};
247    // Match when there is no else block
248    ($if_block:block) => {{
249        #[cfg(feature = "cvm_guest")]
250        {
251            if ::tdx_guest::tdx_is_enabled() {
252                $if_block
253            }
254        }
255    }};
256}
257
258pub use if_tdx_enabled;
ostd/arch/x86/mod.rs

ostd/arch/x86/
mod.rs
