Skip to main content

ostd/arch/x86/kernel/
kvm_clock.rs

1// SPDX-License-Identifier: MPL-2.0
2
3use core::{
4    sync::atomic::{Ordering, fence},
5    time::Duration,
6};
7
8use spin::Once;
9use x86::msr;
10
11use crate::{
12    arch::{cpu::cpuid, io::io_mem::read_once},
13    impl_frame_meta_for,
14    mm::{self, Frame, FrameAllocOptions, HasPaddr},
15};
16
17// KVM feature bits and MSR values.
18// Reference: <https://elixir.bootlin.com/linux/v7.0/source/arch/x86/include/uapi/asm/kvm_para.h>.
19const KVM_FEATURE_CLOCKSOURCE2: u32 = 3;
20const MSR_KVM_SYSTEM_TIME_NEW: u32 = 0x4b56_4d01;
21const MSR_KVM_WALL_CLOCK_NEW: u32 = 0x4b56_4d00;
22const KVM_MSR_ENABLE_BIT: u64 = 1;
23
24const MAX_RETRIES: usize = 1_000_000;
25
26const NANOS_PER_SECOND: u32 = 1_000_000_000;
27
28/// Returns a consistent snapshot read from the KVM pvclock page.
29pub fn read_pvclock_time_snapshot() -> Option<PvclockTimeSnapshot> {
30    PvclockPage::setup()?.read_time_snapshot()
31}
32
33/// Returns the wall-clock epoch recorded in the KVM wall-clock page.
34///
35/// The returned duration is always less than `u32::MAX` seconds, because the
36/// page stores whole seconds as a `u32`.
37pub fn read_wall_clock_epoch() -> Option<Duration> {
38    WallClockPage::setup()?.read_epoch()
39}
40
41/// Determines the TSC frequency from KVM's paravirtual clock.
42pub(super) fn determine_tsc_freq() -> Option<u64> {
43    let pvclock_page = PvclockPage::setup()?;
44    let time_snapshot = pvclock_page.read_time_snapshot()?;
45    time_snapshot.tsc_freq_hz()
46}
47
48/// Returns whether the current environment supports the KVM clocksource v2.
49fn has_kvm_clocksource2() -> bool {
50    cpuid::query_if_running_under_kvm() && cpuid::query_kvm_feature(KVM_FEATURE_CLOCKSOURCE2)
51}
52
53/// Shared version-based protocol for reading KVM pvclock data consistently.
54///
55/// An odd `version` means the host is still writing the pvclock fields.
56/// A read is consistent only when the version observed before and after
57/// reading the fields is the same even value.
58/// Reference: <https://elixir.bootlin.com/linux/v7.0/source/Documentation/virt/kvm/x86/msr.rst#L86-L90>.
59///
60/// FIXME: All synchronization in this protocol is based on the assumption that
61/// we are performing atomic operations under the same memory model as the KVM
62/// side. However:
63/// 1. The Linux memory model is incompatible with Rust's.
64/// 2. We use non-atomic volatile operations to safely communicate with
65///    untrusted parties. But these operations cannot establish synchronization.
66trait PvclockVersion {
67    /// Returns the current pvclock `version` value.
68    fn read_version(&self) -> u32;
69
70    /// Begins a consistent read and returns the version to compare against.
71    fn read_begin(&self) -> u32 {
72        // Masking the odd bit makes `is_version_changed` reject both reads that begin
73        // during an update and reads that race with a completed update.
74        let version = self.read_version() & !1;
75
76        // Synchronize with the (even) version update so that we can see the data written before
77        // the version is written.
78        fence(Ordering::Acquire);
79        version
80    }
81
82    /// Returns whether the pvclock `version` changed since the read was begun.
83    fn is_version_changed(&self, version: u32) -> bool {
84        // Synchronize with the data update so that we can see the (odd) version update
85        // before the data is written (if the data is changed concurrently).
86        fence(Ordering::Acquire);
87        self.read_version() != version
88    }
89}
90
91/// A handle to the KVM pvclock page shared with the hypervisor.
92struct PvclockPage(Frame<PvclockPageMeta>);
93
94impl PvclockPage {
95    /// Sets up the KVM pvclock page and returns a handle to it.
96    fn setup() -> Option<&'static Self> {
97        static PVCLOCK_PAGE: Once<PvclockPage> = Once::new();
98
99        if !has_kvm_clocksource2() {
100            return None;
101        }
102
103        if let Some(page) = PVCLOCK_PAGE.get() {
104            return Some(page);
105        }
106
107        let frame = FrameAllocOptions::new()
108            .alloc_frame_with(PvclockPageMeta)
109            .ok()?;
110        Some(PVCLOCK_PAGE.call_once(|| {
111            // SAFETY: `frame` is a live page that will be retained by
112            // `PVCLOCK_PAGE`, and this MSR is supported per
113            // `has_kvm_clocksource2()` above.
114            unsafe {
115                msr::wrmsr(
116                    MSR_KVM_SYSTEM_TIME_NEW,
117                    frame.paddr() as u64 | KVM_MSR_ENABLE_BIT,
118                )
119            };
120            Self(frame)
121        }))
122    }
123
124    /// Returns a consistent snapshot read from the KVM pvclock page.
125    fn read_time_snapshot(&self) -> Option<PvclockTimeSnapshot> {
126        for _ in 0..MAX_RETRIES {
127            let version = self.read_begin();
128
129            let tsc_timestamp = self.read_tsc_timestamp();
130            let system_time = self.read_system_time();
131            let tsc_to_system_mul = self.read_tsc_to_system_mul();
132            let tsc_shift = self.read_tsc_shift();
133
134            if !self.is_version_changed(version) && tsc_to_system_mul != 0 {
135                return Some(PvclockTimeSnapshot {
136                    tsc_timestamp,
137                    system_time,
138                    tsc_to_system_mul,
139                    tsc_shift,
140                });
141            }
142
143            core::hint::spin_loop();
144        }
145
146        None
147    }
148
149    fn read_tsc_timestamp(&self) -> u64 {
150        // SAFETY: `self.as_ptr()` points to a live KVM pvclock page.
151        unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).tsc_timestamp)) }
152    }
153
154    fn read_system_time(&self) -> u64 {
155        // SAFETY: `self.as_ptr()` points to a live KVM pvclock page.
156        unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).system_time)) }
157    }
158
159    fn read_tsc_to_system_mul(&self) -> u32 {
160        // SAFETY: `self.as_ptr()` points to a live KVM pvclock page.
161        unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).tsc_to_system_mul)) }
162    }
163
164    fn read_tsc_shift(&self) -> i8 {
165        // SAFETY: `self.as_ptr()` points to a live KVM pvclock page.
166        unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).tsc_shift)) }
167    }
168
169    /// Returns a pointer that points to the pvclock page.
170    fn as_ptr(&self) -> *const PvclockVcpuTimeInfo {
171        mm::paddr_to_vaddr(self.0.paddr()) as *const PvclockVcpuTimeInfo
172    }
173}
174
175impl PvclockVersion for PvclockPage {
176    fn read_version(&self) -> u32 {
177        // SAFETY: `self.as_ptr()` points to a live KVM pvclock page.
178        unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).version)) }
179    }
180}
181
182/// A consistent snapshot read from the KVM pvclock page.
183#[derive(Clone, Copy, Debug)]
184pub struct PvclockTimeSnapshot {
185    /// The TSC value at the time the snapshot was taken.
186    pub tsc_timestamp: u64,
187    /// The system time in nanoseconds at `tsc_timestamp`.
188    pub system_time: u64,
189    /// The multiplier for converting TSC ticks to system time.
190    pub tsc_to_system_mul: u32,
191    /// The shift for converting TSC ticks to system time.
192    pub tsc_shift: i8,
193}
194
195impl PvclockTimeSnapshot {
196    /// Returns the TSC frequency in Hz derived from this snapshot.
197    fn tsc_freq_hz(&self) -> Option<u64> {
198        // The pvclock ABI encodes the TSC frequency using a scale factor
199        // (`tsc_to_system_mul`) and a shift (`tsc_shift`) instead of a direct
200        // frequency.
201        // Reference: <https://elixir.bootlin.com/linux/v7.0/source/arch/x86/kernel/pvclock.c#L27>.
202
203        let base_khz = (1_000_000u64 << 32).checked_div(u64::from(self.tsc_to_system_mul))?;
204
205        let tsc_khz = if self.tsc_shift < 0 {
206            base_khz.checked_shl((self.tsc_shift as i32).unsigned_abs())?
207        } else {
208            base_khz.checked_shr(self.tsc_shift as u32)?
209        };
210
211        let freq = tsc_khz.checked_mul(1000)?;
212        (freq != 0).then_some(freq)
213    }
214}
215
216/// A handle to the KVM wall clock page shared with the hypervisor.
217struct WallClockPage(Frame<WallClockPageMeta>);
218
219impl WallClockPage {
220    /// Sets up the KVM wall clock page and returns a handle to it.
221    fn setup() -> Option<&'static Self> {
222        static WALL_CLOCK_PAGE: Once<WallClockPage> = Once::new();
223
224        if !has_kvm_clocksource2() {
225            return None;
226        }
227
228        if let Some(page) = WALL_CLOCK_PAGE.get() {
229            return Some(page);
230        }
231
232        let frame = FrameAllocOptions::new()
233            .alloc_frame_with(WallClockPageMeta)
234            .ok()?;
235        Some(WALL_CLOCK_PAGE.call_once(|| {
236            // SAFETY: `frame` is a live page that will be retained by
237            // `WALL_CLOCK_PAGE`, and this MSR is supported per
238            // `has_kvm_clocksource2()` above.
239            unsafe { msr::wrmsr(MSR_KVM_WALL_CLOCK_NEW, frame.paddr() as u64) };
240            Self(frame)
241        }))
242    }
243
244    /// Returns the wall-clock epoch recorded in this page, as a duration since
245    /// the Unix epoch.
246    fn read_epoch(&self) -> Option<Duration> {
247        for _ in 0..MAX_RETRIES {
248            let version = self.read_begin();
249
250            let sec = self.read_sec();
251            let nsec = self.read_nsec();
252
253            if !self.is_version_changed(version) {
254                if nsec >= NANOS_PER_SECOND {
255                    return None;
256                }
257                return Some(Duration::new(u64::from(sec), nsec));
258            }
259
260            core::hint::spin_loop();
261        }
262
263        None
264    }
265
266    fn read_sec(&self) -> u32 {
267        // SAFETY: `self.as_ptr()` points to a live KVM wall clock page.
268        unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).sec)) }
269    }
270
271    fn read_nsec(&self) -> u32 {
272        // SAFETY: `self.as_ptr()` points to a live KVM wall clock page.
273        unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).nsec)) }
274    }
275
276    /// Returns a pointer that points to the wall clock page.
277    fn as_ptr(&self) -> *const PvclockWallClock {
278        mm::paddr_to_vaddr(self.0.paddr()) as *const PvclockWallClock
279    }
280}
281
282impl PvclockVersion for WallClockPage {
283    fn read_version(&self) -> u32 {
284        // SAFETY: `self.as_ptr()` points to a live KVM wall clock page.
285        unsafe { read_once(core::ptr::addr_of!((*self.as_ptr()).version)) }
286    }
287}
288
289/// Guest-visible layout of the KVM pvclock page (`struct pvclock_vcpu_time_info` in Linux).
290///
291/// Reference: <https://elixir.bootlin.com/linux/v7.0/source/arch/x86/include/asm/pvclock-abi.h#L26>.
292#[repr(C, packed)]
293struct PvclockVcpuTimeInfo {
294    version: u32,
295    _pad0: u32,
296    tsc_timestamp: u64,
297    system_time: u64,
298    tsc_to_system_mul: u32,
299    tsc_shift: i8,
300    _flags: u8,
301    _pad: [u8; 2],
302}
303
304/// Guest-visible layout of the KVM wall clock page (`struct pvclock_wall_clock` in Linux).
305///
306/// Reference: <https://elixir.bootlin.com/linux/v7.0/source/arch/x86/include/asm/pvclock-abi.h#L37>.
307#[repr(C, packed)]
308struct PvclockWallClock {
309    version: u32,
310    sec: u32,
311    nsec: u32,
312}
313
314/// Metadata for the KVM pvclock frame.
315///
316/// The frame is written by the hypervisor and read by the guest clock code,
317/// so it is kept typed to prevent OSTD users from reading or writing it as
318/// untyped memory via [`VmReader`] and [`VmWriter`].
319///
320/// [`VmReader`]: crate::mm::VmReader
321/// [`VmWriter`]: crate::mm::VmWriter
322struct PvclockPageMeta;
323impl_frame_meta_for!(PvclockPageMeta);
324
325/// Metadata for the KVM wall clock frame.
326///
327/// The frame is written by the hypervisor and read by the guest clock code,
328/// so it is kept typed to prevent OSTD users from reading or writing it as
329/// untyped memory via [`VmReader`] and [`VmWriter`].
330///
331/// [`VmReader`]: crate::mm::VmReader
332/// [`VmWriter`]: crate::mm::VmWriter
333struct WallClockPageMeta;
334impl_frame_meta_for!(WallClockPageMeta);