ostd/io/io_mem/
mod.rs

1// SPDX-License-Identifier: MPL-2.0
2
3//! I/O memory and its allocator that allocates memory I/O (MMIO) to device drivers.
4
5mod allocator;
6pub(crate) mod util;
7
8use core::{
9    marker::PhantomData,
10    ops::{Deref, Range},
11};
12
13use align_ext::AlignExt;
14use inherit_methods_macro::inherit_methods;
15
16pub(crate) use self::allocator::IoMemAllocatorBuilder;
17pub(super) use self::allocator::init;
18#[cfg(all(target_arch = "x86_64", feature = "cvm_guest"))]
19use crate::arch::{if_tdx_enabled, tdx_guest::unprotect_gpa_tdvm_call};
20use crate::{
21    Error,
22    arch::io::io_mem::{read_once, write_once},
23    cpu::{AtomicCpuSet, CpuSet},
24    mm::{
25        Fallible, Infallible, PAGE_SIZE, PodOnce, VmIo, VmIoFill, VmIoOnce, VmReader, VmWriter,
26        io::{
27            Io,
28            copy::{memcpy, memset},
29        },
30        kspace::kvirt_area::KVirtArea,
31        page_prop::{CachePolicy, PageFlags, PageProperty, PrivilegedPageFlags},
32        tlb::{TlbFlushOp, TlbFlusher},
33    },
34    prelude::*,
35    task::disable_preempt,
36};
37
38/// A marker type used for [`IoMem`],
39/// representing that the underlying MMIO is used for security-sensitive operations.
40#[derive(Clone, Debug)]
41pub(crate) enum Sensitive {}
42
43/// A marker type used for [`IoMem`],
44/// representing that the underlying MMIO is used for security-insensitive operations.
45#[derive(Clone, Debug)]
46pub enum Insensitive {}
47
48/// I/O memory.
49#[derive(Clone, Debug)]
50pub struct IoMem<SecuritySensitivity = Insensitive> {
51    kvirt_area: Arc<KVirtArea>,
52    // The actually used range for MMIO is `kvirt_area.start + offset..kvirt_area.start + offset + limit`
53    offset: usize,
54    limit: usize,
55    pa: Paddr,
56    cache_policy: CachePolicy,
57    phantom: PhantomData<SecuritySensitivity>,
58}
59
60impl<SecuritySensitivity> IoMem<SecuritySensitivity> {
61    /// Slices the `IoMem`, returning another `IoMem` representing the subslice.
62    ///
63    /// # Panics
64    ///
65    /// This method will panic if the range is empty or out of bounds.
66    pub fn slice(&self, range: Range<usize>) -> Self {
67        // This ensures `range.start < range.end` and `range.end <= limit`.
68        assert!(!range.is_empty() && range.end <= self.limit);
69
70        // We've checked the range is in bounds, so we can construct the new `IoMem` safely.
71        Self {
72            kvirt_area: self.kvirt_area.clone(),
73            offset: self.offset + range.start,
74            limit: range.len(),
75            pa: self.pa + range.start,
76            cache_policy: self.cache_policy,
77            phantom: PhantomData,
78        }
79    }
80
81    /// Creates a new `IoMem`.
82    ///
83    /// # Safety
84    ///
85    /// 1. This function must be called after the kernel page table is activated.
86    /// 2. The given physical address range must be in the I/O memory region.
87    /// 3. Reading from or writing to I/O memory regions may have side effects.
88    ///    If `SecuritySensitivity` is `Insensitive`, those side effects must
89    ///    not cause soundness problems (e.g., they must not corrupt the kernel
90    ///    memory).
91    pub(crate) unsafe fn new(range: Range<Paddr>, flags: PageFlags, cache: CachePolicy) -> Self {
92        let first_page_start = range.start.align_down(PAGE_SIZE);
93        let last_page_end = range.end.align_up(PAGE_SIZE);
94
95        let frames_range = first_page_start..last_page_end;
96        let area_size = frames_range.len();
97
98        #[cfg(target_arch = "x86_64")]
99        let priv_flags = if_tdx_enabled!({
100            assert!(
101                first_page_start == range.start && last_page_end == range.end,
102                "I/O memory is not page aligned, which cannot be unprotected in TDX: {:#x?}..{:#x?}",
103                range.start,
104                range.end,
105            );
106
107            // SAFETY:
108            //  - The range `first_page_start..last_page_end` is always page aligned.
109            //  - FIXME: We currently do not limit the I/O memory allocator with the maximum GPA,
110            //    so the address range may not fall in the GPA limit.
111            //  - The caller guarantees that operations on the I/O memory do not have any side
112            //    effects that may cause soundness problems, so the pages can safely be viewed as
113            //    untyped memory.
114            unsafe { unprotect_gpa_tdvm_call(first_page_start, area_size).unwrap() };
115
116            PrivilegedPageFlags::SHARED
117        } else {
118            PrivilegedPageFlags::empty()
119        });
120        #[cfg(not(target_arch = "x86_64"))]
121        let priv_flags = PrivilegedPageFlags::empty();
122
123        let prop = PageProperty {
124            flags,
125            cache,
126            priv_flags,
127        };
128
129        let kva = {
130            // SAFETY: The caller of `IoMem::new()` ensures that the given
131            // physical address range is I/O memory, so it is safe to map.
132            let kva = unsafe { KVirtArea::map_untracked_frames(area_size, 0, frames_range, prop) };
133
134            let target_cpus = AtomicCpuSet::new(CpuSet::new_full());
135            let mut flusher = TlbFlusher::new(&target_cpus, disable_preempt());
136            flusher.issue_tlb_flush(TlbFlushOp::for_range(kva.range()));
137            flusher.dispatch_tlb_flush();
138            flusher.sync_tlb_flush();
139
140            kva
141        };
142
143        Self {
144            kvirt_area: Arc::new(kva),
145            offset: range.start - first_page_start,
146            limit: range.len(),
147            pa: range.start,
148            cache_policy: cache,
149            phantom: PhantomData,
150        }
151    }
152
153    /// Returns the cache policy of this `IoMem`.
154    pub fn cache_policy(&self) -> CachePolicy {
155        self.cache_policy
156    }
157
158    /// Returns the base virtual address of the MMIO range.
159    fn base(&self) -> usize {
160        self.kvirt_area.deref().start() + self.offset
161    }
162
163    /// Validates that the offset range lies within the MMIO window.
164    fn check_range(&self, offset: usize, len: usize) -> Result<()> {
165        if offset.checked_add(len).is_none_or(|end| end > self.limit) {
166            return Err(Error::InvalidArgs);
167        }
168        Ok(())
169    }
170}
171
172#[cfg_attr(
173    any(target_arch = "loongarch64", target_arch = "aarch64"),
174    expect(unused)
175)]
176impl IoMem<Sensitive> {
177    /// Reads a value of the `PodOnce` type at the specified offset using one
178    /// non-tearing memory load.
179    ///
180    /// Except that the offset is specified explicitly, the semantics of this
181    /// method is the same as [`VmReader::read_once`].
182    ///
183    /// # Safety
184    ///
185    /// The caller must ensure that the offset and the read operation is valid,
186    /// e.g., follows the specification when used for implementing drivers, does
187    /// not cause any out-of-bounds access, and does not cause unsound side
188    /// effects (e.g., corrupting the kernel memory).
189    pub(crate) unsafe fn read_once<T: PodOnce>(&self, offset: usize) -> T {
190        debug_assert!(offset + size_of::<T>() <= self.limit);
191        let ptr = (self.kvirt_area.deref().start() + self.offset + offset) as *const T;
192        // SAFETY: The safety of the read operation's semantics is upheld by the caller.
193        unsafe { read_once(ptr) }
194    }
195
196    /// Writes a value of the `PodOnce` type at the specified offset using one
197    /// non-tearing memory store.
198    ///
199    /// Except that the offset is specified explicitly, the semantics of this
200    /// method is the same as [`VmWriter::write_once`].
201    ///
202    /// # Safety
203    ///
204    /// The caller must ensure that the offset and the write operation is valid,
205    /// e.g., follows the specification when used for implementing drivers, does
206    /// not cause any out-of-bounds access, and does not cause unsound side
207    /// effects (e.g., corrupting the kernel memory).
208    pub(crate) unsafe fn write_once<T: PodOnce>(&self, offset: usize, value: &T) {
209        debug_assert!(offset + size_of::<T>() <= self.limit);
210        let ptr = (self.kvirt_area.deref().start() + self.offset + offset) as *mut T;
211        // SAFETY: The safety of the write operation's semantics is upheld by the caller.
212        unsafe { write_once(ptr, *value) };
213    }
214}
215
216impl IoMem<Insensitive> {
217    /// Acquires an `IoMem` instance for the given range.
218    ///
219    /// The I/O memory cache policy is set to uncacheable by default.
220    pub fn acquire(range: Range<Paddr>) -> Result<IoMem<Insensitive>> {
221        Self::acquire_with_cache_policy(range, CachePolicy::Uncacheable)
222    }
223
224    /// Acquires an `IoMem` instance for the given range with the specified cache policy.
225    pub fn acquire_with_cache_policy(
226        range: Range<Paddr>,
227        cache_policy: CachePolicy,
228    ) -> Result<IoMem<Insensitive>> {
229        allocator::IO_MEM_ALLOCATOR
230            .get()
231            .unwrap()
232            .acquire(range, cache_policy)
233            .ok_or(Error::AccessDenied)
234    }
235
236    /// Reads from MMIO into fallible memory and returns the copied length.
237    ///
238    /// This method performs the same low-level copy primitive as [`VmIo::read`],
239    /// but exposes partial progress instead of enforcing no-short-read semantics.
240    pub fn read_fallible(
241        &self,
242        offset: usize,
243        writer: &mut VmWriter,
244    ) -> Result<usize, (Error, usize)> {
245        let len = writer.avail();
246        self.check_range(offset, len).map_err(|err| (err, 0))?;
247
248        let src = (self.base() + offset) as *const u8;
249        // SAFETY: `src` points to a validated MMIO range and `writer.cursor()` points to
250        // fallible destination memory tracked by `writer`.
251        let copied = unsafe { memcpy::<Fallible, Io>(writer.cursor(), src, len) };
252        writer.skip(copied);
253
254        if copied < len {
255            Err((Error::PageFault, copied))
256        } else {
257            Ok(copied)
258        }
259    }
260
261    /// Writes from fallible memory to MMIO and returns the copied length.
262    ///
263    /// This method performs the same low-level copy primitive as [`VmIo::write`],
264    /// but exposes partial progress instead of enforcing no-short-write semantics.
265    pub fn write_fallible(
266        &self,
267        offset: usize,
268        reader: &mut VmReader,
269    ) -> Result<usize, (Error, usize)> {
270        let len = reader.remain();
271        self.check_range(offset, len).map_err(|err| (err, 0))?;
272
273        let dst = (self.base() + offset) as *mut u8;
274        // SAFETY: `dst` points to a validated MMIO range and `reader.cursor()` points to
275        // fallible source memory tracked by `reader`.
276        let copied = unsafe { memcpy::<Io, Fallible>(dst, reader.cursor(), len) };
277        reader.skip(copied);
278
279        if copied < len {
280            Err((Error::PageFault, copied))
281        } else {
282            Ok(copied)
283        }
284    }
285}
286
287impl VmIoOnce for IoMem<Insensitive> {
288    fn read_once<T: PodOnce>(&self, offset: usize) -> Result<T> {
289        self.check_range(offset, size_of::<T>())?;
290        let ptr = (self.base() + offset) as *const T;
291        if !ptr.is_aligned() {
292            return Err(Error::InvalidArgs);
293        }
294
295        // SAFETY: The pointer is properly aligned and within the validated range.
296        let val = unsafe { read_once(ptr) };
297        Ok(val)
298    }
299
300    fn write_once<T: PodOnce>(&self, offset: usize, value: &T) -> Result<()> {
301        self.check_range(offset, size_of::<T>())?;
302        let ptr = (self.base() + offset) as *mut T;
303        if !ptr.is_aligned() {
304            return Err(Error::InvalidArgs);
305        }
306
307        // SAFETY: The pointer is properly aligned and within the validated range.
308        unsafe { write_once(ptr, *value) };
309        Ok(())
310    }
311}
312
313impl VmIo for IoMem<Insensitive> {
314    fn read(&self, offset: usize, writer: &mut VmWriter) -> Result<()> {
315        let len = writer.avail();
316        self.check_range(offset, len)?;
317
318        let src = (self.base() + offset) as *const u8;
319        // SAFETY: `src` points to a validated MMIO range and `writer.cursor()` points to
320        // fallible destination memory tracked by `writer`.
321        let copied = unsafe { memcpy::<Fallible, Io>(writer.cursor(), src, len) };
322        if copied < len {
323            return Err(Error::PageFault);
324        }
325
326        writer.skip(copied);
327        Ok(())
328    }
329
330    fn read_bytes(&self, offset: usize, buf: &mut [u8]) -> Result<()> {
331        let len = buf.len();
332        self.check_range(offset, len)?;
333        let src = (self.base() + offset) as *const u8;
334        let dst = buf.as_mut_ptr();
335
336        // SAFETY: The `dst` and `src` buffers are valid to write and read for `len` bytes.
337        unsafe { memcpy::<Infallible, Io>(dst, src, len) };
338        Ok(())
339    }
340
341    fn write(&self, offset: usize, reader: &mut VmReader) -> Result<()> {
342        let len = reader.remain();
343        self.check_range(offset, len)?;
344
345        let dst = (self.base() + offset) as *mut u8;
346        // SAFETY: `dst` points to a validated MMIO range and `reader.cursor()` points to
347        // fallible source memory tracked by `reader`.
348        let copied = unsafe { memcpy::<Io, Fallible>(dst, reader.cursor(), len) };
349        if copied < len {
350            return Err(Error::PageFault);
351        }
352
353        reader.skip(copied);
354        Ok(())
355    }
356
357    fn write_bytes(&self, offset: usize, buf: &[u8]) -> Result<()> {
358        let len = buf.len();
359        self.check_range(offset, len)?;
360        let src = buf.as_ptr();
361        let dst = (self.base() + offset) as *mut u8;
362
363        // SAFETY: The `dst` and `src` buffers are valid to write and read for `len` bytes.
364        unsafe { memcpy::<Io, Infallible>(dst, src, len) };
365        Ok(())
366    }
367}
368
369impl VmIoFill for IoMem<Insensitive> {
370    fn fill_zeros(&self, offset: usize, len: usize) -> Result<(), (Error, usize)> {
371        if len == 0 {
372            return Ok(());
373        }
374
375        if offset > self.limit {
376            return Err((Error::InvalidArgs, 0));
377        }
378
379        let available = self.limit - offset;
380        let write_len = core::cmp::min(len, available);
381        if write_len == 0 {
382            return Err((Error::InvalidArgs, 0));
383        }
384
385        let dst = (self.base() + offset) as *mut u8;
386        // SAFETY: `dst` points to the validated MMIO subrange of `write_len` bytes.
387        unsafe { memset::<Io>(dst, 0u8, write_len) };
388
389        if write_len == len {
390            Ok(())
391        } else {
392            Err((Error::InvalidArgs, write_len))
393        }
394    }
395}
396
397macro_rules! impl_vm_io_pointer {
398    ($ty:ty, $from:tt) => {
399        #[inherit_methods(from = $from)]
400        impl VmIo for $ty {
401            fn read(&self, offset: usize, writer: &mut VmWriter) -> Result<()>;
402            fn write(&self, offset: usize, reader: &mut VmReader) -> Result<()>;
403        }
404
405        #[inherit_methods(from = $from)]
406        impl VmIoOnce for $ty {
407            fn read_once<T: PodOnce>(&self, offset: usize) -> Result<T>;
408            fn write_once<T: PodOnce>(&self, offset: usize, value: &T) -> Result<()>;
409        }
410
411        #[inherit_methods(from = $from)]
412        impl VmIoFill for $ty {
413            fn fill_zeros(&self, offset: usize, len: usize) -> Result<(), (Error, usize)>;
414        }
415    };
416}
417
418impl_vm_io_pointer!(&IoMem<Insensitive>, "(**self)");
419impl_vm_io_pointer!(&mut IoMem<Insensitive>, "(**self)");
420
421impl<SecuritySensitivity> HasPaddr for IoMem<SecuritySensitivity> {
422    fn paddr(&self) -> Paddr {
423        self.pa
424    }
425}
426
427impl<SecuritySensitivity> HasSize for IoMem<SecuritySensitivity> {
428    fn size(&self) -> usize {
429        self.limit
430    }
431}
432
433impl<SecuritySensitivity> Drop for IoMem<SecuritySensitivity> {
434    fn drop(&mut self) {
435        // TODO: Multiple `IoMem` instances should not overlap, we should refactor the driver code and
436        // remove the `Clone` and `IoMem::slice`. After refactoring, the `Drop` can be implemented to recycle
437        // the `IoMem`.
438    }
439}
440
441#[cfg(ktest)]
442mod test {
443    use crate::{
444        arch::io::io_mem::{copy_from_mmio, copy_to_mmio, read_once, write_once},
445        prelude::ktest,
446    };
447
448    #[ktest]
449    fn read_write_u8() {
450        let mut data: u8 = 0;
451        // SAFETY: `data` is valid for a single MMIO read/write.
452        unsafe {
453            write_once(&mut data, 42u8);
454            assert_eq!(read_once(&data), 42u8);
455        }
456    }
457
458    #[ktest]
459    fn read_write_u16() {
460        let mut data: u16 = 0;
461        let val: u16 = 0x1234;
462        // SAFETY: `data` is valid for a single MMIO read/write.
463        unsafe {
464            write_once(&mut data, val);
465            assert_eq!(read_once(&data), val);
466        }
467    }
468
469    #[ktest]
470    fn read_write_u32() {
471        let mut data: u32 = 0;
472        let val: u32 = 0x12345678;
473        // SAFETY: `data` is valid for a single MMIO read/write.
474        unsafe {
475            write_once(&mut data, val);
476            assert_eq!(read_once(&data), val);
477        }
478    }
479
480    #[ktest]
481    fn read_write_u64() {
482        let mut data: u64 = 0;
483        let val: u64 = 0xDEADBEEFCAFEBABE;
484        // SAFETY: `data` is valid for a single MMIO read/write.
485        unsafe {
486            write_once(&mut data, val);
487            assert_eq!(read_once(&data), val);
488        }
489    }
490
491    #[ktest]
492    fn boundary_overlap() {
493        let mut data: [u8; 2] = [0xAA, 0xBB];
494        // SAFETY: `data` is valid for a single MMIO read/write.
495        unsafe {
496            write_once(&mut data[0], 0x11u8);
497            assert_eq!(data[0], 0x11);
498            assert_eq!(data[1], 0xBB);
499        }
500    }
501
502    fn fill_pattern(buf: &mut [u8]) {
503        for (idx, byte) in buf.iter_mut().enumerate() {
504            *byte = (idx as u8).wrapping_mul(3).wrapping_add(1);
505        }
506    }
507
508    fn run_copy_from_case(src_offset: usize, dst_offset: usize, len: usize) {
509        let mut src = [0u8; 64];
510        let mut dst = [0u8; 64];
511        fill_pattern(&mut src);
512
513        // SAFETY: Offsets are validated by callers before this helper is invoked.
514        let src_ptr = unsafe { src.as_ptr().add(src_offset) };
515        // SAFETY: Offsets are validated by callers before this helper is invoked.
516        let dst_ptr = unsafe { dst.as_mut_ptr().add(dst_offset) };
517
518        // SAFETY: The test buffers are valid for the requested range.
519        unsafe { copy_from_mmio(dst_ptr, src_ptr, len) };
520
521        assert_eq!(
522            &dst[dst_offset..dst_offset + len],
523            &src[src_offset..src_offset + len]
524        );
525    }
526
527    fn run_copy_to_case(src_offset: usize, dst_offset: usize, len: usize) {
528        let mut src = [0u8; 64];
529        let mut dst = [0u8; 64];
530        fill_pattern(&mut src);
531
532        // SAFETY: Offsets are validated by callers before this helper is invoked.
533        let src_ptr = unsafe { src.as_ptr().add(src_offset) };
534        // SAFETY: Offsets are validated by callers before this helper is invoked.
535        let dst_ptr = unsafe { dst.as_mut_ptr().add(dst_offset) };
536
537        // SAFETY: The test buffers are valid for the requested range.
538        unsafe { copy_to_mmio(src_ptr, dst_ptr, len) };
539
540        assert_eq!(
541            &dst[dst_offset..dst_offset + len],
542            &src[src_offset..src_offset + len]
543        );
544    }
545
546    #[ktest]
547    fn copy_from_alignment_and_sizes() {
548        let word_size = size_of::<usize>();
549        let sizes = [
550            0,
551            1,
552            word_size.saturating_sub(1),
553            word_size,
554            word_size + 1,
555            word_size * 2 + 3,
556        ];
557        let offsets = [0, 1, 2];
558
559        for &len in &sizes {
560            for &src_offset in &offsets {
561                for &dst_offset in &offsets {
562                    if src_offset + len <= 64 && dst_offset + len <= 64 {
563                        run_copy_from_case(src_offset, dst_offset, len);
564                    }
565                }
566            }
567        }
568    }
569
570    #[ktest]
571    fn copy_to_alignment_and_sizes() {
572        let word_size = size_of::<usize>();
573        let sizes = [
574            0,
575            1,
576            word_size.saturating_sub(1),
577            word_size,
578            word_size + 1,
579            word_size * 2 + 3,
580        ];
581        let offsets = [0, 1, 2];
582
583        for &len in &sizes {
584            for &src_offset in &offsets {
585                for &dst_offset in &offsets {
586                    if src_offset + len <= 64 && dst_offset + len <= 64 {
587                        run_copy_to_case(src_offset, dst_offset, len);
588                    }
589                }
590            }
591        }
592    }
593}
ostd/io/io_mem/mod.rs

ostd/io/io_mem/
mod.rs
