ostd/arch/x86/
mod.rs

1// SPDX-License-Identifier: MPL-2.0
2
3//! Platform-specific code for the x86 platform.
4
5pub(crate) mod boot;
6pub mod cpu;
7pub mod device;
8pub(crate) mod io;
9pub(crate) mod iommu;
10pub mod irq;
11pub mod kernel;
12pub(crate) mod mm;
13pub mod power;
14pub mod serial;
15pub(crate) mod task;
16mod timer;
17pub mod trap;
18pub mod vm;
19
20#[cfg(feature = "cvm_guest")]
21pub(crate) mod tdx_guest;
22
23#[cfg(feature = "cvm_guest")]
24use crate::mm::frame::unaccepted;
25
26#[cfg(feature = "cvm_guest")]
27pub(crate) fn init_cvm_guest(boot_params: &linux_boot_params::BootParams) {
28    use ::tdx_guest::{
29        SeptVeError, disable_sept_ve, init_tdx, metadata, reduce_unnecessary_ve,
30        tdcall::{InitError, write_td_metadata},
31        tdvmcall::report_fatal_error_simple,
32    };
33
34    match init_tdx() {
35        Ok(td_info) => {
36            reduce_unnecessary_ve().unwrap();
37            match disable_sept_ve(td_info.attributes) {
38                Ok(_) => {}
39                Err(SeptVeError::Misconfiguration) => {
40                    crate::early_println!(
41                        "[kernel] Error: TD misconfiguration: \
42                        The SEPT_VE_DISABLE bit of the TD attributes must be set by VMM \
43                        when running in non-debug mode and FLEXIBLE_PENDING_VE is not enabled."
44                    );
45                    report_fatal_error_simple("TD misconfiguration: SEPT #VE has to be disabled");
46                }
47                Err(e) => {
48                    crate::early_println!("[kernel] Error: Unexpected TDX error: {:?}", e);
49                    report_fatal_error_simple(
50                        "Disabling SEPT #VE failed due to unexpected TDX error",
51                    );
52                }
53            }
54            // Enable notification for zero step attack detection.
55            write_td_metadata(metadata::NOTIFY_ENABLES, 1, 1).unwrap();
56
57            crate::early_println!(
58                "[kernel] Intel TDX initialized\n[kernel] td gpaw: {}, td attributes: {:?}",
59                td_info.gpaw,
60                td_info.attributes
61            );
62            unaccepted::init(boot_params);
63        }
64        Err(InitError::TdxGetVpInfoError(td_call_error)) => {
65            crate::early_println!(
66                "[kernel] Intel TDX not initialized, Failed to get TD info. TD call error: {:?}",
67                td_call_error
68            );
69            report_fatal_error_simple("Intel TDX not initialized, Failed to get TD info.");
70        }
71        // The machine has no TDX support.
72        Err(_) => {}
73    }
74}
75
76/// Architecture-specific initialization on the bootstrapping processor.
77///
78/// It should be called when the heap and frame allocators are available.
79///
80/// # Safety
81///
82/// 1. This function must be called only once in the boot context of the
83///    bootstrapping processor.
84/// 2. This function must be called after the kernel page table is activated on
85///    the bootstrapping processor.
86pub(crate) unsafe fn late_init_on_bsp() {
87    // SAFETY: This is only called once on this BSP in the boot context.
88    unsafe { trap::init_on_cpu() };
89
90    // SAFETY: The caller ensures that this function is only called once on BSP,
91    // after the kernel page table is activated.
92    let io_mem_builder = unsafe { io::construct_io_mem_allocator_builder() };
93
94    kernel::apic::init(&io_mem_builder).expect("APIC doesn't exist");
95    irq::chip::init(&io_mem_builder);
96    irq::ipi::init();
97
98    kernel::tsc::init_tsc_freq();
99    timer::init_on_bsp();
100
101    // SAFETY: We're on the BSP and we're ready to boot all APs.
102    unsafe { crate::boot::smp::boot_all_aps() };
103
104    if_tdx_enabled!({
105        // SAFETY: This is only called once on this BSP in the boot context. APs have been booted.
106        unsafe { unaccepted::accept_memory_on_bsp() };
107    });
108
109    if_tdx_enabled!({
110    } else {
111        match iommu::init(&io_mem_builder) {
112            Ok(_) => {}
113            Err(err) => crate::warn!("IOMMU initialization error: {:?}", err),
114        }
115    });
116
117    // SAFETY:
118    // 1. All the system device memory have been removed from the builder.
119    // 2. All the port I/O regions belonging to the system device are defined using the macros.
120    // 3. `MAX_IO_PORT` defined in `crate::arch::io` is the maximum value specified by x86-64.
121    unsafe { crate::io::init(io_mem_builder) };
122
123    kernel::acpi::init();
124    power::init();
125}
126
127/// Initializes application-processor-specific state.
128///
129/// # Safety
130///
131/// 1. This function must be called only once on each application processor.
132/// 2. This function must be called after the BSP's call to [`late_init_on_bsp`]
133///    and before any other architecture-specific code in this module is called
134///    on this AP.
135pub(crate) unsafe fn init_on_ap() {
136    timer::init_on_ap();
137}
138
139/// Returns the frequency of TSC. The unit is Hz.
140pub fn tsc_freq() -> u64 {
141    use core::sync::atomic::Ordering;
142
143    kernel::tsc::TSC_FREQ.load(Ordering::Acquire)
144}
145
146/// Reads the current value of the processor's time-stamp counter (TSC).
147pub fn read_tsc() -> u64 {
148    use core::arch::x86_64::_rdtsc;
149
150    // SAFETY: It is safe to read a time-related counter.
151    unsafe { _rdtsc() }
152}
153
154/// Reads a hardware generated 64-bit random value.
155///
156/// Returns `None` if no random value was generated.
157pub fn read_random() -> Option<u64> {
158    use core::arch::x86_64::_rdrand64_step;
159
160    use cpu::extension::{IsaExtensions, has_extensions};
161
162    if !has_extensions(IsaExtensions::RDRAND) {
163        return None;
164    }
165
166    // Recommendation from "Intel(R) Digital Random Number Generator (DRNG) Software
167    // Implementation Guide" - Section 5.2.1 and "Intel(R) 64 and IA-32 Architectures
168    // Software Developer's Manual" - Volume 1 - Section 7.3.17.1.
169    const RETRY_LIMIT: usize = 10;
170
171    for _ in 0..RETRY_LIMIT {
172        let mut val = 0;
173        let generated = unsafe { _rdrand64_step(&mut val) };
174        if generated == 1 {
175            return Some(val);
176        }
177    }
178    None
179}
180
181pub(crate) fn enable_cpu_features() {
182    use cpu::extension::{IsaExtensions, has_extensions};
183    use x86_64::registers::{
184        control::{Cr0Flags, Cr4Flags},
185        xcontrol::XCr0Flags,
186    };
187
188    cpu::extension::init();
189    vm::init();
190
191    let mut cr0 = x86_64::registers::control::Cr0::read();
192    cr0 |= Cr0Flags::WRITE_PROTECT;
193    // These FPU control bits should be set for new CPUs (e.g., all CPUs with 64-bit support) and
194    // modern OSes. See recommendation from "Intel(R) 64 and IA-32 Architectures Software
195    // Developer's Manual" - Volume 3 - Section 10.2.1, Configuring the x87 FPU Environment.
196    cr0 |= Cr0Flags::NUMERIC_ERROR | Cr0Flags::MONITOR_COPROCESSOR;
197    unsafe { x86_64::registers::control::Cr0::write(cr0) };
198
199    let mut cr4 = x86_64::registers::control::Cr4::read();
200    cr4 |= Cr4Flags::OSFXSR | Cr4Flags::OSXMMEXCPT_ENABLE | Cr4Flags::PAGE_GLOBAL;
201    if has_extensions(IsaExtensions::XSAVE) {
202        cr4 |= Cr4Flags::OSXSAVE;
203    }
204    // For now, we unconditionally require the `rdfsbase`, `wrfsbase`, `rdgsbase`, and `wrgsbase`
205    // instructions because they are used when switching contexts, getting the address of a
206    // CPU-local variable, e.t.c. Meanwhile, this is at a very early stage of the boot process, so
207    // we want to avoid failing immediately even if we cannot enable these instructions (though the
208    // kernel will certainly fail later when they are absent).
209    //
210    // Note that this also enables the userspace to control their own FS/GS bases, which requires
211    // the kernel to properly deal with the arbitrary base values set by the userspace program.
212    if has_extensions(IsaExtensions::FSGSBASE) {
213        cr4 |= Cr4Flags::FSGSBASE;
214    }
215    unsafe { x86_64::registers::control::Cr4::write(cr4) };
216
217    if has_extensions(IsaExtensions::XSAVE) {
218        let mut xcr0 = x86_64::registers::xcontrol::XCr0::read();
219        xcr0 |= XCr0Flags::SSE;
220        if has_extensions(IsaExtensions::AVX) {
221            xcr0 |= XCr0Flags::AVX;
222        }
223        if has_extensions(IsaExtensions::AVX512F) {
224            xcr0 |= XCr0Flags::OPMASK | XCr0Flags::ZMM_HI256 | XCr0Flags::HI16_ZMM;
225        }
226        unsafe { x86_64::registers::xcontrol::XCr0::write(xcr0) };
227    }
228
229    cpu::context::enable_essential_features();
230
231    mm::enable_essential_features();
232}
233
234/// Inserts a TDX-specific code block.
235///
236/// This macro conditionally executes a TDX-specific code block based on the following conditions:
237/// (1) The `cvm_guest` feature is enabled at compile time.
238/// (2) The TDX feature is detected at runtime via `::tdx_guest::tdx_is_enabled()`.
239///
240/// If both conditions are met, the `if_block` is executed. If an `else_block` is provided, it will be executed
241/// when either the `cvm_guest` feature is not enabled or the TDX feature is not detected at runtime.
242#[macro_export]
243macro_rules! if_tdx_enabled {
244    // Match when there is an else block
245    ($if_block:block else $else_block:block) => {{
246        #[cfg(feature = "cvm_guest")]
247        {
248            if ::tdx_guest::tdx_is_enabled() {
249                $if_block
250            } else {
251                $else_block
252            }
253        }
254        #[cfg(not(feature = "cvm_guest"))]
255        {
256            $else_block
257        }
258    }};
259    // Match when there is no else block
260    ($if_block:block) => {{
261        #[cfg(feature = "cvm_guest")]
262        {
263            if ::tdx_guest::tdx_is_enabled() {
264                $if_block
265            }
266        }
267    }};
268}
269
270pub use if_tdx_enabled;
ostd/arch/x86/mod.rs

ostd/arch/x86/
mod.rs
