Skip to main content

ostd/mm/dma/
dma_coherent.rs

1// SPDX-License-Identifier: MPL-2.0
2use vstd::prelude::*;
3#[cfg(feature = "irc11")]
4use vstd::thread_view::Objective;
5use vstd_extra::{
6    atomic_data::AtomicDataWithOwner,
7    ownership::{Inv, OwnerOf},
8    resource_invariant::SimpleResourceInvariant,
9};
10
11use core::marker::PhantomData;
12
13use crate::{
14    error::Error,
15    mm::{
16        HasPaddr, Paddr,
17        dma::{Daddr, DmaType, dma_type},
18        frame::{Segment, untyped::AnyUFrameMeta},
19        io::{
20            FallibleVmRead, FallibleVmWrite, Infallible, PodOnce, VmIo, VmIoMemView, VmIoOnce,
21            VmIoOwner, VmReader, VmWriter,
22        },
23        kspace::{KERNEL_BASE_VADDR, KERNEL_END_VADDR, VMALLOC_BASE_VADDR},
24        paddr_to_vaddr,
25    },
26    specs::{arch::*, mm::virt_mem::VirtPtr},
27    sync::{PreemptDisabled, RwArc, RwLockReadGuard},
28};
29
30use super::{DmaError, HasDaddr, check_and_insert_dma_mapping, is_valid_daddr};
31
32verus! {
33
34/// A coherent (or consistent) DMA mapping,
35/// which guarantees that the device and the CPU can
36/// access the data in parallel.
37///
38/// The mapping will be destroyed automatically when
39/// the object is dropped.
40pub struct DmaCoherent<M: AnyUFrameMeta + ?Sized> {
41    pub inner: RwArc<DmaCoherentInnerAtomic<M>>,
42}
43
44/// The tracked owner for the inner part of a [`DmaCoherent`].
45///
46/// This mirrors [`DmaStreamVmIoOwner`](super::dma_stream::DmaStreamVmIoOwner):
47/// the real memory ownership lives in the inner `RwArc`, and this token is
48/// only the public `VmIo` permission parameter.
49pub struct DmaCoherentVmIoOwner<M: ?Sized>(pub PhantomData<M>);
50
51/// The inner part of a [`DmaCoherent`].
52pub struct DmaCoherentInner<M: AnyUFrameMeta + ?Sized> {
53    /// Verus does not support trait objects here, so the metadata type is
54    /// represented explicitly by `M`.
55    pub segment: Segment<M>,
56    pub start_daddr: Daddr,
57    /// TODO: restore cache-policy switching once the page-table API is verified.
58    #[expect(unused)]
59    pub is_cache_coherent: bool,
60}
61
62/// The owner of the inner part of a [`DmaCoherent`].
63pub tracked struct DmaCoherentInnerOwner<M: AnyUFrameMeta + ?Sized> {
64    pub _marker: PhantomData<M>,
65}
66
67// This owner contains only PhantomData and carries no subjective memory permissions.
68#[cfg(feature = "irc11")]
69unsafe impl<M: AnyUFrameMeta + ?Sized> Objective for DmaCoherentInnerOwner<M> {
70
71}
72
73pub ghost struct DmaCoherentInnerInvariant;
74
75pub type DmaCoherentInnerAtomic<M> = AtomicDataWithOwner<
76    DmaCoherentInner<M>,
77    DmaCoherentInnerInvariant,
78>;
79
80impl<M: AnyUFrameMeta + ?Sized> Inv for DmaCoherent<M> {
81    open spec fn inv(self) -> bool {
82        true
83    }
84}
85
86impl<M: AnyUFrameMeta + ?Sized> Inv for DmaCoherentInner<M> {
87    open spec fn inv(self) -> bool {
88        &&& self.segment.inv()
89        &&& is_valid_daddr(self.start_daddr)
90    }
91}
92
93impl<M: AnyUFrameMeta + ?Sized> Inv for DmaCoherentInnerOwner<M> {
94    open spec fn inv(self) -> bool {
95        true
96    }
97}
98
99impl<M: AnyUFrameMeta + ?Sized> SimpleResourceInvariant<
100    DmaCoherentInner<M>,
101> for DmaCoherentInnerInvariant {
102    type Resource = DmaCoherentInnerOwner<M>;
103
104    open spec fn inv(value: DmaCoherentInner<M>, resource: DmaCoherentInnerOwner<M>) -> bool {
105        &&& resource.inv()
106        &&& value.inv()
107    }
108}
109
110#[verus_verify]
111impl<M: AnyUFrameMeta + ?Sized + OwnerOf> DmaCoherent<M> {
112    /// Creates a coherent DMA mapping backed by `segment`.
113    ///
114    /// The `is_cache_coherent` argument specifies whether the target device can
115    /// access main memory in a CPU-cache-coherent way.
116    #[verus_spec(r =>
117        requires
118            segment.inv(),
119        ensures
120            r matches Ok(r) ==> r.inner.wf(),
121    )]
122    pub fn map(segment: Segment<M>, is_cache_coherent: bool) -> Result<Self, DmaError> {
123        let start_paddr = segment.start_paddr();
124        let frame_count = segment.size() / PAGE_SIZE;
125
126        if !check_and_insert_dma_mapping(start_paddr, frame_count) {
127            return Err(DmaError::AlreadyMapped);
128        }/*
129        // Original cache policy change (removed during Verus migration):
130        if !is_cache_coherent {
131            let page_table = KERNEL_PAGE_TABLE.get().unwrap();
132            let vaddr = paddr_to_vaddr(start_paddr);
133            let va_range = vaddr..vaddr + (frame_count * PAGE_SIZE);
134            unsafe {
135                page_table
136                    .protect_flush_tlb(&va_range, |p| p.cache = CachePolicy::Uncacheable)
137                    .unwrap();
138            }
139        }
140        */
141
142        let start_daddr = match dma_type() {
143            DmaType::Direct => {
144                /*
145                // Original TDX unprotect (removed during Verus migration):
146                #[cfg(target_arch = "x86_64")]
147                crate::arch::if_tdx_enabled!({
148                    unsafe {
149                        tdx_guest::unprotect_gpa_range(start_paddr, frame_count).unwrap();
150                    }
151                });
152                */
153                start_paddr as Daddr
154            },
155            DmaType::Iommu => {
156                for i in 0..frame_count
157                    invariant
158                        segment.inv(),
159                        start_paddr == segment.start_paddr(),
160                        frame_count == segment.size() / PAGE_SIZE,
161                {
162                    assume(start_paddr + i * PAGE_SIZE <= usize::MAX);
163                    let _paddr = start_paddr + i * PAGE_SIZE;
164                    /*
165                    // Original IOMMU map (removed during Verus migration):
166                    unsafe {
167                        iommu::map(paddr as Daddr, paddr).unwrap();
168                    }
169                    */
170                }
171
172                start_paddr as Daddr
173            },
174        };
175
176        let tracked inner_owner = DmaCoherentInnerOwner { _marker: PhantomData::<M> };
177
178        let inner = RwArc::new(
179            AtomicDataWithOwner::new(
180                DmaCoherentInner { segment, start_daddr, is_cache_coherent },
181                Tracked(inner_owner),
182                Ghost(DmaCoherentInnerInvariant),
183            ),
184        );
185
186        Ok(DmaCoherent { inner })
187    }
188
189    /// Returns the starting device address from a read guard.
190    #[inline(always)]
191    #[verus_spec(r =>
192        requires
193            this@.inv(),
194        returns
195            this@.data.start_daddr,
196    )]
197    fn daddr_inner(this: &RwLockReadGuard<DmaCoherentInnerAtomic<M>, PreemptDisabled>) -> Daddr {
198        this.start_daddr
199    }
200
201    /// Returns the starting physical address from a read guard.
202    #[inline(always)]
203    #[verus_spec(r =>
204        requires
205            this@.inv(),
206        returns
207            this@.data.segment.start_paddr(),
208    )]
209    fn paddr_inner(this: &RwLockReadGuard<DmaCoherentInnerAtomic<M>, PreemptDisabled>) -> Paddr {
210        this.segment.start_paddr()
211    }
212
213    /// Returns the number of frames from a read guard.
214    #[inline(always)]
215    #[verus_spec(r =>
216        requires
217            this@.inv(),
218        returns
219            this@.data.segment.size() / PAGE_SIZE,
220    )]
221    fn nframes_inner(this: &RwLockReadGuard<DmaCoherentInnerAtomic<M>, PreemptDisabled>) -> usize {
222        this.segment.size() / PAGE_SIZE
223    }
224
225    /// Returns the number of bytes from a read guard.
226    #[inline(always)]
227    #[verus_spec(r =>
228        requires
229            this@.inv(),
230        returns
231            this@.data.segment.size(),
232    )]
233    fn nbytes_inner(this: &RwLockReadGuard<DmaCoherentInnerAtomic<M>, PreemptDisabled>) -> usize {
234        this.segment.size()
235    }
236
237    /// Returns a reader to read data from it using a read guard.
238    #[inline(always)]
239    #[verus_spec(r =>
240        with
241            -> reader_owner: Tracked<VmIoOwner>,
242        requires
243            this@.inv(),
244        ensures
245            r.inv(),
246            reader_owner@.inv(),
247            reader_owner@.is_kernel,
248            reader_owner@.has_read_view(),
249            reader_owner@.read_view_initialized(),
250            r.wf(reader_owner@),
251            KERNEL_BASE_VADDR <= r.cursor.range@.start,
252            r.cursor.range@.end <= KERNEL_END_VADDR,
253    )]
254    fn reader_inner<'a>(
255        this: RwLockReadGuard<'a, DmaCoherentInnerAtomic<M>, PreemptDisabled>,
256    ) -> VmReader<'a, Infallible> {
257        proof_decl! {
258            let ghost id: nat;
259            let tracked mut reader_owner: VmIoOwner;
260        }
261        proof {
262            lemma_max_paddr_range();
263            lemma_paddr_to_vaddr_properties(this@.data.segment.start_paddr());
264        }
265
266        let vaddr = paddr_to_vaddr(this.segment.start_paddr());
267        let len = this.segment.size();
268        let ghost range = vaddr..(vaddr + len) as usize;
269        let ptr = VirtPtr { vaddr, range: Ghost(range) };
270        proof {
271            assert(KERNEL_BASE_VADDR > 0) by (compute_only);
272            assert(vaddr > 0);
273            assert(VMALLOC_BASE_VADDR <= KERNEL_END_VADDR) by (compute_only);
274            assert(ptr.inv());
275        }
276
277        let reader = unsafe {
278            proof_with!(Ghost(id) => Tracked(reader_owner));
279            VmReader::from_kernel_space(ptr, len)
280        };
281        proof_with!(|= Tracked(reader_owner));
282        reader
283    }
284
285    /// Returns a writer to write data to it using a read guard.
286    #[inline(always)]
287    #[verus_spec(r =>
288        with
289            -> writer_owner: Tracked<VmIoOwner>,
290        requires
291            this@.inv(),
292        ensures
293            r.inv(),
294            writer_owner@.inv(),
295            writer_owner@.is_kernel,
296            writer_owner@.has_write_view(),
297            r.wf(writer_owner@),
298            KERNEL_BASE_VADDR <= r.cursor.range@.start,
299            r.cursor.range@.end <= KERNEL_END_VADDR,
300    )]
301    fn writer_inner<'a>(
302        this: RwLockReadGuard<'a, DmaCoherentInnerAtomic<M>, PreemptDisabled>,
303    ) -> VmWriter<'a, Infallible> {
304        proof_decl! {
305            let ghost id: nat;
306            let tracked mut writer_owner: VmIoOwner;
307        }
308        proof {
309            lemma_max_paddr_range();
310            lemma_paddr_to_vaddr_properties(this@.data.segment.start_paddr());
311        }
312
313        let vaddr = paddr_to_vaddr(this.segment.start_paddr());
314        let len = this.segment.size();
315        let ghost range = vaddr..(vaddr + len) as usize;
316        let ptr = VirtPtr { vaddr, range: Ghost(range) };
317        proof {
318            assert(KERNEL_BASE_VADDR > 0) by (compute_only);
319            assert(vaddr > 0);
320            assert(VMALLOC_BASE_VADDR <= KERNEL_END_VADDR) by (compute_only);
321            assert(ptr.inv());
322        }
323
324        let writer = unsafe {
325            proof_with!(Ghost(id), Tracked(false) => Tracked(writer_owner));
326            VmWriter::from_kernel_space(ptr, len)
327        };
328        proof_with!(|= Tracked(writer_owner));
329        writer
330    }
331}
332
333impl<M: AnyUFrameMeta + ?Sized> DmaCoherent<M> {
334    /// Acquires a read guard for the inner DMA-coherent state.
335    #[inline(always)]
336    #[verifier::external_body]
337    #[verus_spec(r =>
338        ensures
339            self.inner.wf(),
340            r@.inv(),
341    )]
342    pub fn read_inner(&self) -> RwLockReadGuard<'_, DmaCoherentInnerAtomic<M>, PreemptDisabled> {
343        self.inner.read()
344    }
345}
346
347#[verus_verify]
348impl<M: AnyUFrameMeta + ?Sized + OwnerOf> DmaCoherent<M> {
349    /// Returns the starting device address.
350    #[inline(always)]
351    #[verus_spec(r =>
352        requires
353            self.inner.wf(),
354        ensures
355            self.inner.wf(),
356    )]
357    pub fn daddr(&self) -> Daddr {
358        let inner = self.read_inner();
359        Self::daddr_inner(&inner)
360    }
361
362    /// Returns the starting physical address.
363    #[inline(always)]
364    #[verus_spec(r =>
365        requires
366            self.inner.wf(),
367        ensures
368            self.inner.wf(),
369    )]
370    pub fn paddr(&self) -> Paddr {
371        let inner = self.read_inner();
372        Self::paddr_inner(&inner)
373    }
374
375    /// Returns the number of frames.
376    #[inline(always)]
377    #[verus_spec(r =>
378        requires
379            self.inner.wf(),
380        ensures
381            self.inner.wf(),
382    )]
383    pub fn nframes(&self) -> usize {
384        let inner = self.read_inner();
385        Self::nframes_inner(&inner)
386    }
387
388    /// Returns the number of bytes in the DMA mapping.
389    #[inline(always)]
390    #[verus_spec(r =>
391        requires
392            self.inner.wf(),
393        ensures
394            self.inner.wf(),
395    )]
396    pub fn nbytes(&self) -> usize {
397        let inner = self.read_inner();
398        Self::nbytes_inner(&inner)
399    }
400
401    /// Returns a reader to read data from it.
402    #[inline(always)]
403    #[verus_spec(r =>
404        with
405            -> reader_perm: Tracked<VmIoOwner>,
406        requires
407            self.inner.wf(),
408        ensures
409            self.inner.wf(),
410            r.inv(),
411            reader_perm@.inv(),
412            reader_perm@.is_kernel,
413            reader_perm@.has_read_view(),
414            r.wf(reader_perm@),
415            KERNEL_BASE_VADDR <= r.cursor.range@.start,
416            r.cursor.range@.end <= KERNEL_END_VADDR,
417    )]
418    pub fn reader<'a>(&'a self) -> VmReader<'a, Infallible> {
419        let inner = self.read_inner();
420        proof_with!(=> Tracked(reader_perm));
421        let reader = Self::reader_inner(inner);
422        proof_with!(|= Tracked(reader_perm));
423        VmReader {
424            ghost_id: reader.ghost_id,
425            cursor: reader.cursor,
426            end: reader.end,
427            phantom: PhantomData,
428        }
429    }
430
431    /// Returns a writer to write data into it.
432    #[inline(always)]
433    #[verus_spec(r =>
434        with
435            -> writer_perm: Tracked<VmIoOwner>,
436        requires
437            self.inner.wf(),
438        ensures
439            self.inner.wf(),
440            r.inv(),
441            writer_perm@.inv(),
442            writer_perm@.is_kernel,
443            writer_perm@.has_write_view(),
444            r.wf(writer_perm@),
445    )]
446    pub fn writer<'a>(&'a self) -> VmWriter<'a, Infallible> {
447        let inner = self.read_inner();
448        proof_with!(=> Tracked(writer_perm));
449        let writer = Self::writer_inner(inner);
450        proof_with!(|= Tracked(writer_perm));
451        VmWriter {
452            ghost_id: writer.ghost_id,
453            cursor: writer.cursor,
454            end: writer.end,
455            phantom: PhantomData,
456        }
457    }
458}
459
460#[verus_verify]
461impl<M: AnyUFrameMeta + ?Sized> Clone for DmaCoherent<M> {
462    #[verus_spec(
463        returns
464            self,
465    )]
466    fn clone(&self) -> Self {
467        DmaCoherent { inner: self.inner.clone() }
468    }
469}
470
471impl<M: AnyUFrameMeta + ?Sized + OwnerOf> HasDaddr for DmaCoherent<M> {
472    #[inline]
473    fn daddr(&self) -> Daddr {
474        let inner = self.read_inner();
475        Self::daddr_inner(&inner)
476    }
477}
478
479impl<M: AnyUFrameMeta + ?Sized + OwnerOf> HasPaddr for DmaCoherent<M> {
480    #[inline]
481    fn paddr(&self) -> Paddr {
482        let inner = self.read_inner();
483        Self::paddr_inner(&inner)
484    }
485}
486
487/*
488// Original Deref impl (removed during Verus migration):
489impl Deref for DmaCoherent {
490    type Target = USegment;
491    fn deref(&self) -> &Self::Target {
492        &self.inner.segment
493    }
494}
495*/
496
497/*
498// Original Drop impl for DmaCoherentInner (removed during Verus migration):
499impl Drop for DmaCoherentInner {
500    fn drop(&mut self) {
501        let start_paddr = self.segment.start_paddr();
502        let frame_count = self.segment.size() / PAGE_SIZE;
503
504        match dma_type() {
505            DmaType::Direct => {
506                #[cfg(target_arch = "x86_64")]
507                crate::arch::if_tdx_enabled!({
508                    unsafe {
509                        tdx_guest::protect_gpa_range(start_paddr, frame_count).unwrap();
510                    }
511                });
512            }
513            DmaType::Iommu => {
514                for i in 0..frame_count {
515                    let paddr = start_paddr + (i * PAGE_SIZE);
516                    iommu::unmap(paddr as Daddr).unwrap();
517                }
518            }
519        }
520
521        if !self.is_cache_coherent {
522            let page_table = KERNEL_PAGE_TABLE.get().unwrap();
523            let vaddr = paddr_to_vaddr(start_paddr);
524            let va_range = vaddr..vaddr + (frame_count * PAGE_SIZE);
525            unsafe {
526                page_table
527                    .protect_flush_tlb(&va_range, |p| p.cache = CachePolicy::Writeback)
528                    .unwrap();
529            }
530        }
531
532        remove_dma_mapping(start_paddr, frame_count);
533    }
534}
535*/
536
537impl<M: AnyUFrameMeta + ?Sized + Send + Sync + OwnerOf> VmIo<
538    DmaCoherentVmIoOwner<M>,
539> for DmaCoherent<M> {
540    closed spec fn obeys_vmio_spec() -> bool {
541        true
542    }
543
544    closed spec fn obeys_vmio_read_requires() -> bool {
545        true
546    }
547
548    closed spec fn obeys_vmio_write_requires() -> bool {
549        true
550    }
551
552    closed spec fn obeys_vmio_read_spec() -> bool {
553        false
554    }
555
556    closed spec fn obeys_vmio_write_spec() -> bool {
557        false
558    }
559
560    open spec fn read_requires(
561        self,
562        offset: usize,
563        writer: VmWriter<'_>,
564        writer_own: VmIoOwner,
565        owner: DmaCoherentVmIoOwner<M>,
566    ) -> bool {
567        &&& self.inv()
568        &&& self.inner.wf()
569        &&& writer.inv()
570        &&& writer_own.inv()
571        &&& writer.wf(writer_own)
572        &&& writer_own.mem_view matches Some(VmIoMemView::WriteView(_))
573        &&& writer.cursor.vaddr > 0
574        &&& (writer.cursor.range@.end <= KERNEL_BASE_VADDR || KERNEL_END_VADDR
575            <= writer.cursor.range@.start)
576    }
577
578    open spec fn write_requires(
579        self,
580        offset: usize,
581        reader: VmReader<'_>,
582        reader_own: VmIoOwner,
583        owner: DmaCoherentVmIoOwner<M>,
584    ) -> bool {
585        &&& self.inv()
586        &&& self.inner.wf()
587        &&& reader.inv()
588        &&& reader_own.inv()
589        &&& reader.wf(reader_own)
590        &&& reader_own.mem_view matches Some(VmIoMemView::ReadView(_))
591        &&& reader_own.read_view_initialized()
592        &&& reader.cursor.vaddr > 0
593        &&& (reader.cursor.range@.end <= KERNEL_BASE_VADDR || KERNEL_END_VADDR
594            <= reader.cursor.range@.start)
595    }
596
597    open spec fn read_spec(
598        self,
599        offset: usize,
600        old_writer: VmWriter<'_>,
601        new_writer: VmWriter<'_>,
602        old_writer_own: VmIoOwner,
603        new_writer_own: VmIoOwner,
604        old_owner: DmaCoherentVmIoOwner<M>,
605        new_owner: DmaCoherentVmIoOwner<M>,
606        r: core::result::Result<(), Error>,
607    ) -> bool {
608        &&& self.inv()
609        &&& new_owner == old_owner
610        &&& new_writer.inv()
611        &&& new_writer_own.inv()
612        &&& new_writer.wf(new_writer_own)
613        &&& match r {
614            Ok(_) => {
615                &&& new_writer.avail_spec() == 0
616                &&& new_writer.cursor.vaddr == old_writer.cursor.vaddr + old_writer.avail_spec()
617                &&& new_writer_own.range.start == old_writer_own.range.start
618                    + old_writer.avail_spec()
619            },
620            Err(_) => {
621                &&& new_writer == old_writer
622                &&& new_writer_own == old_writer_own
623            },
624        }
625    }
626
627    open spec fn write_spec(
628        self,
629        offset: usize,
630        old_reader: VmReader<'_>,
631        new_reader: VmReader<'_>,
632        old_reader_own: VmIoOwner,
633        new_reader_own: VmIoOwner,
634        old_owner: DmaCoherentVmIoOwner<M>,
635        new_owner: DmaCoherentVmIoOwner<M>,
636        r: core::result::Result<(), Error>,
637    ) -> bool {
638        &&& self.inv()
639        &&& new_owner == old_owner
640        &&& new_reader.inv()
641        &&& new_reader_own.inv()
642        &&& new_reader.wf(new_reader_own)
643        &&& match r {
644            Ok(_) => {
645                &&& new_reader.remain_spec() == 0
646                &&& new_reader.cursor.vaddr == old_reader.cursor.vaddr + old_reader.remain_spec()
647                &&& new_reader_own.range.start == old_reader_own.range.start
648                    + old_reader.remain_spec()
649            },
650            Err(_) => {
651                &&& new_reader == old_reader
652                &&& new_reader_own == old_reader_own
653            },
654        }
655    }
656
657    fn read(
658        &self,
659        offset: usize,
660        writer: &mut VmWriter<'_>,
661        Tracked(writer_own): Tracked<&mut VmIoOwner>,
662        Tracked(owner): Tracked<&mut DmaCoherentVmIoOwner<M>>,
663    ) -> core::result::Result<(), Error> {
664        proof {
665            assert(Self::obeys_vmio_spec());
666            assert(Self::obeys_vmio_read_requires());
667            assert(!Self::obeys_vmio_read_spec());
668            assert(Self::read_requires(*self, offset, *writer, *writer_own, *owner));
669        }
670        proof_decl! {
671            let tracked mut reader_own;
672        }
673        let mut reader = {
674            let inner = self.read_inner();
675            #[verus_spec(with => Tracked(reader_own))]
676            Self::reader_inner(inner)
677        };
678
679        let remain_before_skip = reader.remain();
680        if offset > remain_before_skip {
681            return Err(Error::InvalidArgs);
682        }
683        let remain = remain_before_skip - offset;
684        let len = writer.avail();
685        if remain < len {
686            return Err(Error::InvalidArgs);
687        }
688        if len == 0 {
689            proof {
690                assert(writer.avail_spec() == 0);
691            }
692            return Ok(());
693        }
694        let reader = reader.skip(offset);
695        proof {
696            reader_own.advance(offset);
697            assert(reader_own.inv());
698            assert(writer_own.inv());
699            assert(reader_own.mem_view matches Some(VmIoMemView::ReadView(_)));
700            assert(reader_own.read_view_initialized());
701            assert(writer_own.mem_view matches Some(VmIoMemView::WriteView(_)));
702            assert(reader.remain_spec() >= len);
703        }
704
705        let copied = match reader.read_fallible(writer) {
706            Ok(copied) => copied,
707            Err((err, copied)) => {
708                writer.cursor = writer.cursor.sub(copied);
709                return Err(err);
710            },
711        };
712
713        proof {
714            reader_own.advance(copied);
715            writer_own.advance(copied);
716            assert(*owner == *old(owner));
717        }
718
719        Ok(())
720    }
721
722    fn write(
723        &self,
724        offset: usize,
725        reader: &mut VmReader<'_>,
726        Tracked(reader_own): Tracked<&mut VmIoOwner>,
727        Tracked(owner): Tracked<&mut DmaCoherentVmIoOwner<M>>,
728    ) -> core::result::Result<(), Error> {
729        proof {
730            assert(Self::obeys_vmio_spec());
731            assert(Self::obeys_vmio_write_requires());
732            assert(!Self::obeys_vmio_write_spec());
733            assert(Self::write_requires(*self, offset, *reader, *reader_own, *owner));
734        }
735        proof_decl! {
736            let tracked mut writer_own;
737        }
738        let mut writer = {
739            let inner = self.read_inner();
740            #[verus_spec(with => Tracked(writer_own))]
741            Self::writer_inner(inner)
742        };
743
744        let avail_before_skip = writer.avail();
745        if offset > avail_before_skip {
746            return Err(Error::InvalidArgs);
747        }
748        let avail = avail_before_skip - offset;
749        let len = reader.remain();
750        if avail < len {
751            return Err(Error::InvalidArgs);
752        }
753        if len == 0 {
754            proof {
755                assert(reader.remain_spec() == 0);
756            }
757            return Ok(());
758        }
759        let writer = writer.skip(offset);
760        proof {
761            writer_own.advance(offset);
762
763            assert(writer.avail_spec() >= len);
764        }
765        let copied = match writer.write_fallible(reader) {
766            Ok(copied) => copied,
767            Err((err, copied)) => {
768                reader.cursor = reader.cursor.sub(copied);
769                return Err(err);
770            },
771        };
772        proof {
773            writer_own.advance(copied);
774            reader_own.advance(copied);
775            assert(*owner == *old(owner));
776        }
777
778        Ok(())
779    }
780}
781
782impl<M: AnyUFrameMeta + ?Sized + Send + Sync + OwnerOf> VmIoOnce for DmaCoherent<M> {
783    closed spec fn obeys_vmio_once_read_requires() -> bool {
784        true
785    }
786
787    closed spec fn obeys_vmio_once_write_requires() -> bool {
788        true
789    }
790
791    closed spec fn obeys_vmio_once_read_ensures() -> bool {
792        true
793    }
794
795    closed spec fn obeys_vmio_once_write_ensures() -> bool {
796        true
797    }
798
799    #[verifier::external_body]
800    fn read_once<T: PodOnce>(&self, offset: usize) -> core::result::Result<T, Error> {
801        self.reader().skip(offset).read_once()
802    }
803
804    #[verifier::external_body]
805    fn write_once<T: PodOnce>(&self, offset: usize, new_val: &T) -> core::result::Result<
806        (),
807        Error,
808    > {
809        self.writer().skip(offset).write_once(new_val)
810    }
811}
812
813} // verus!