ostd/arch/x86/kernel/kvm_clock.rs
1// SPDX-License-Identifier: MPL-2.0
2
3use core::{
4 sync::atomic::{Ordering, fence},
5 time::Duration,
6};
7
8use spin::Once;
9use x86::msr;
10
11use crate::{
12 arch::{cpu::cpuid, io::io_mem::read_once},
13 impl_frame_meta_for,
14 mm::{self, Frame, FrameAllocOptions, HasPaddr},
15};
16
17// KVM feature bits and MSR values.
18// Reference: <https://elixir.bootlin.com/linux/v7.0/source/arch/x86/include/uapi/asm/kvm_para.h>.
19const KVM_FEATURE_CLOCKSOURCE2: u32 = 3;
20const MSR_KVM_SYSTEM_TIME_NEW: u32 = 0x4b56_4d01;
21const MSR_KVM_WALL_CLOCK_NEW: u32 = 0x4b56_4d00;
22const KVM_MSR_ENABLE_BIT: u64 = 1;
23
24const MAX_RETRIES: usize = 1_000_000;
25
26const NANOS_PER_SECOND: u32 = 1_000_000_000;
27
28/// Returns a consistent snapshot read from the KVM pvclock page.
29pub fn read_pvclock_time_snapshot() -> Option<PvclockTimeSnapshot> {
30 PvclockPage::setup()?.read_time_snapshot()
31}
32
33/// Returns the wall-clock epoch recorded in the KVM wall-clock page.
34///
35/// The returned duration is always less than `u32::MAX` seconds, because the
36/// page stores whole seconds as a `u32`.
37pub fn read_wall_clock_epoch() -> Option<Duration> {
38 WallClockPage::setup()?.read_epoch()
39}
40
41/// Determines the TSC frequency from KVM's paravirtual clock.
42pub(super) fn determine_tsc_freq() -> Option<u64> {
43 let pvclock_page = PvclockPage::setup()?;
44 let time_snapshot = pvclock_page.read_time_snapshot()?;
45 time_snapshot.tsc_freq_hz()
46}
47
48/// Returns whether the current environment supports the KVM clocksource v2.
49fn has_kvm_clocksource2() -> bool {
50 cpuid::query_if_running_under_kvm() && cpuid::query_kvm_feature(KVM_FEATURE_CLOCKSOURCE2)
51}
52
53/// Shared version-based protocol for reading KVM pvclock data consistently.
54///
55/// An odd `version` means the host is still writing the pvclock fields.
56/// A read is consistent only when the version observed before and after
57/// reading the fields is the same even value.
58/// Reference: <https://elixir.bootlin.com/linux/v7.0/source/Documentation/virt/kvm/x86/msr.rst#L86-L90>.
59///
60/// FIXME: All synchronization in this protocol is based on the assumption that
61/// we are performing atomic operations under the same memory model as the KVM
62/// side. However:
63/// 1. The Linux memory model is incompatible with Rust's.
64/// 2. We use non-atomic volatile operations to safely communicate with
65/// untrusted parties. But these operations cannot establish synchronization.
66trait PvclockVersion {
67 /// Returns the current pvclock `version` value.
68 fn read_version(&self) -> u32;
69
70 /// Begins a consistent read and returns the version to compare against.
71 fn read_begin(&self) -> u32 {
72 // Masking the odd bit makes `is_version_changed` reject both reads that begin
73 // during an update and reads that race with a completed update.
74 let version = self.read_version() & !1;
75
76 // Synchronize with the (even) version update so that we can see the data written before
77 // the version is written.
78 fence(Ordering::Acquire);
79 version
80 }
81
82 /// Returns whether the pvclock `version` changed since the read was begun.
83 fn is_version_changed(&self, version: u32) -> bool {
84 // Synchronize with the data update so that we can see the (odd) version update
85 // before the data is written (if the data is changed concurrently).
86 fence(Ordering::Acquire);
87 self.read_version() != version
88 }
89}
90
91/// A handle to the KVM pvclock page shared with the hypervisor.
92struct PvclockPage(Frame<PvclockPageMeta>);
93
94impl PvclockPage {
95 /// Sets up the KVM pvclock page and returns a handle to it.
96 fn setup() -> Option<&'static Self> {
97 static PVCLOCK_PAGE: Once<PvclockPage> = Once::new();
98
99 if !has_kvm_clocksource2() {
100 return None;
101 }
102
103 if let Some(page) = PVCLOCK_PAGE.get() {
104 return Some(page);
105 }
106
107 let frame = FrameAllocOptions::new()
108 .alloc_frame_with(PvclockPageMeta)
109 .ok()?;
110 Some(PVCLOCK_PAGE.call_once(|| {
111 // SAFETY: `frame` is a live page that will be retained by
112 // `PVCLOCK_PAGE`, and this MSR is supported per
113 // `has_kvm_clocksource2()` above.
114 unsafe {
115 msr::wrmsr(
116 MSR_KVM_SYSTEM_TIME_NEW,
117 frame.paddr() as u64 | KVM_MSR_ENABLE_BIT,
118 )
119 };
120 Self(frame)
121 }))
122 }
123
124 /// Returns a consistent snapshot read from the KVM pvclock page.
125 fn read_time_snapshot(&self) -> Option<PvclockTimeSnapshot> {
126 for _ in 0..MAX_RETRIES {
127 let version = self.read_begin();
128
129 let tsc_timestamp = self.read_tsc_timestamp();
130 let system_time = self.read_system_time();
131 let tsc_to_system_mul = self.read_tsc_to_system_mul();
132 let tsc_shift = self.read_tsc_shift();
133
134 if !self.is_version_changed(version) && tsc_to_system_mul != 0 {
135 return Some(PvclockTimeSnapshot {
136 tsc_timestamp,
137 system_time,
138 tsc_to_system_mul,
139 tsc_shift,
140 });
141 }
142
143 core::hint::spin_loop();
144 }
145
146 None
147 }
148
149 fn read_tsc_timestamp(&self) -> u64 {
150 // SAFETY: `self.as_ptr()` points to a live KVM pvclock page.
151 unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).tsc_timestamp)) }
152 }
153
154 fn read_system_time(&self) -> u64 {
155 // SAFETY: `self.as_ptr()` points to a live KVM pvclock page.
156 unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).system_time)) }
157 }
158
159 fn read_tsc_to_system_mul(&self) -> u32 {
160 // SAFETY: `self.as_ptr()` points to a live KVM pvclock page.
161 unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).tsc_to_system_mul)) }
162 }
163
164 fn read_tsc_shift(&self) -> i8 {
165 // SAFETY: `self.as_ptr()` points to a live KVM pvclock page.
166 unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).tsc_shift)) }
167 }
168
169 /// Returns a pointer that points to the pvclock page.
170 fn as_ptr(&self) -> *const PvclockVcpuTimeInfo {
171 mm::paddr_to_vaddr(self.0.paddr()) as *const PvclockVcpuTimeInfo
172 }
173}
174
175impl PvclockVersion for PvclockPage {
176 fn read_version(&self) -> u32 {
177 // SAFETY: `self.as_ptr()` points to a live KVM pvclock page.
178 unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).version)) }
179 }
180}
181
182/// A consistent snapshot read from the KVM pvclock page.
183#[derive(Clone, Copy, Debug)]
184pub struct PvclockTimeSnapshot {
185 /// The TSC value at the time the snapshot was taken.
186 pub tsc_timestamp: u64,
187 /// The system time in nanoseconds at `tsc_timestamp`.
188 pub system_time: u64,
189 /// The multiplier for converting TSC ticks to system time.
190 pub tsc_to_system_mul: u32,
191 /// The shift for converting TSC ticks to system time.
192 pub tsc_shift: i8,
193}
194
195impl PvclockTimeSnapshot {
196 /// Returns the TSC frequency in Hz derived from this snapshot.
197 fn tsc_freq_hz(&self) -> Option<u64> {
198 // The pvclock ABI encodes the TSC frequency using a scale factor
199 // (`tsc_to_system_mul`) and a shift (`tsc_shift`) instead of a direct
200 // frequency.
201 // Reference: <https://elixir.bootlin.com/linux/v7.0/source/arch/x86/kernel/pvclock.c#L27>.
202
203 let base_khz = (1_000_000u64 << 32).checked_div(u64::from(self.tsc_to_system_mul))?;
204
205 let tsc_khz = if self.tsc_shift < 0 {
206 base_khz.checked_shl((self.tsc_shift as i32).unsigned_abs())?
207 } else {
208 base_khz.checked_shr(self.tsc_shift as u32)?
209 };
210
211 let freq = tsc_khz.checked_mul(1000)?;
212 (freq != 0).then_some(freq)
213 }
214}
215
216/// A handle to the KVM wall clock page shared with the hypervisor.
217struct WallClockPage(Frame<WallClockPageMeta>);
218
219impl WallClockPage {
220 /// Sets up the KVM wall clock page and returns a handle to it.
221 fn setup() -> Option<&'static Self> {
222 static WALL_CLOCK_PAGE: Once<WallClockPage> = Once::new();
223
224 if !has_kvm_clocksource2() {
225 return None;
226 }
227
228 if let Some(page) = WALL_CLOCK_PAGE.get() {
229 return Some(page);
230 }
231
232 let frame = FrameAllocOptions::new()
233 .alloc_frame_with(WallClockPageMeta)
234 .ok()?;
235 Some(WALL_CLOCK_PAGE.call_once(|| {
236 // SAFETY: `frame` is a live page that will be retained by
237 // `WALL_CLOCK_PAGE`, and this MSR is supported per
238 // `has_kvm_clocksource2()` above.
239 unsafe { msr::wrmsr(MSR_KVM_WALL_CLOCK_NEW, frame.paddr() as u64) };
240 Self(frame)
241 }))
242 }
243
244 /// Returns the wall-clock epoch recorded in this page, as a duration since
245 /// the Unix epoch.
246 fn read_epoch(&self) -> Option<Duration> {
247 for _ in 0..MAX_RETRIES {
248 let version = self.read_begin();
249
250 let sec = self.read_sec();
251 let nsec = self.read_nsec();
252
253 if !self.is_version_changed(version) {
254 if nsec >= NANOS_PER_SECOND {
255 return None;
256 }
257 return Some(Duration::new(u64::from(sec), nsec));
258 }
259
260 core::hint::spin_loop();
261 }
262
263 None
264 }
265
266 fn read_sec(&self) -> u32 {
267 // SAFETY: `self.as_ptr()` points to a live KVM wall clock page.
268 unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).sec)) }
269 }
270
271 fn read_nsec(&self) -> u32 {
272 // SAFETY: `self.as_ptr()` points to a live KVM wall clock page.
273 unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).nsec)) }
274 }
275
276 /// Returns a pointer that points to the wall clock page.
277 fn as_ptr(&self) -> *const PvclockWallClock {
278 mm::paddr_to_vaddr(self.0.paddr()) as *const PvclockWallClock
279 }
280}
281
282impl PvclockVersion for WallClockPage {
283 fn read_version(&self) -> u32 {
284 // SAFETY: `self.as_ptr()` points to a live KVM wall clock page.
285 unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).version)) }
286 }
287}
288
289/// Guest-visible layout of the KVM pvclock page (`struct pvclock_vcpu_time_info` in Linux).
290///
291/// Reference: <https://elixir.bootlin.com/linux/v7.0/source/arch/x86/include/asm/pvclock-abi.h#L26>.
292#[repr(C, packed)]
293struct PvclockVcpuTimeInfo {
294 version: u32,
295 _pad0: u32,
296 tsc_timestamp: u64,
297 system_time: u64,
298 tsc_to_system_mul: u32,
299 tsc_shift: i8,
300 _flags: u8,
301 _pad: [u8; 2],
302}
303
304/// Guest-visible layout of the KVM wall clock page (`struct pvclock_wall_clock` in Linux).
305///
306/// Reference: <https://elixir.bootlin.com/linux/v7.0/source/arch/x86/include/asm/pvclock-abi.h#L37>.
307#[repr(C, packed)]
308struct PvclockWallClock {
309 version: u32,
310 sec: u32,
311 nsec: u32,
312}
313
314/// Metadata for the KVM pvclock frame.
315///
316/// The frame is written by the hypervisor and read by the guest clock code,
317/// so it is kept typed to prevent OSTD users from reading or writing it as
318/// untyped memory via [`VmReader`] and [`VmWriter`].
319///
320/// [`VmReader`]: crate::mm::VmReader
321/// [`VmWriter`]: crate::mm::VmWriter
322struct PvclockPageMeta;
323impl_frame_meta_for!(PvclockPageMeta);
324
325/// Metadata for the KVM wall clock frame.
326///
327/// The frame is written by the hypervisor and read by the guest clock code,
328/// so it is kept typed to prevent OSTD users from reading or writing it as
329/// untyped memory via [`VmReader`] and [`VmWriter`].
330///
331/// [`VmReader`]: crate::mm::VmReader
332/// [`VmWriter`]: crate::mm::VmWriter
333struct WallClockPageMeta;
334impl_frame_meta_for!(WallClockPageMeta);