1use vstd::prelude::*;
3#[cfg(feature = "irc11")]
4use vstd::thread_view::Objective;
5use vstd_extra::{
6 atomic_data::AtomicDataWithOwner,
7 ownership::{Inv, OwnerOf},
8 resource_invariant::SimpleResourceInvariant,
9};
10
11use core::marker::PhantomData;
12
13use crate::{
14 error::Error,
15 mm::{
16 HasPaddr, Paddr,
17 dma::{Daddr, DmaType, dma_type},
18 frame::{Segment, untyped::AnyUFrameMeta},
19 io::{
20 FallibleVmRead, FallibleVmWrite, Infallible, PodOnce, VmIo, VmIoMemView, VmIoOnce,
21 VmIoOwner, VmReader, VmWriter,
22 },
23 kspace::{KERNEL_BASE_VADDR, KERNEL_END_VADDR, VMALLOC_BASE_VADDR},
24 paddr_to_vaddr,
25 },
26 specs::{arch::*, mm::virt_mem::VirtPtr},
27 sync::{PreemptDisabled, RwArc, RwLockReadGuard},
28};
29
30use super::{DmaError, HasDaddr, check_and_insert_dma_mapping, is_valid_daddr};
31
32verus! {
33
34pub struct DmaCoherent<M: AnyUFrameMeta + ?Sized> {
41 pub inner: RwArc<DmaCoherentInnerAtomic<M>>,
42}
43
44pub struct DmaCoherentVmIoOwner<M: ?Sized>(pub PhantomData<M>);
50
51pub struct DmaCoherentInner<M: AnyUFrameMeta + ?Sized> {
53 pub segment: Segment<M>,
56 pub start_daddr: Daddr,
57 #[expect(unused)]
59 pub is_cache_coherent: bool,
60}
61
62pub tracked struct DmaCoherentInnerOwner<M: AnyUFrameMeta + ?Sized> {
64 pub _marker: PhantomData<M>,
65}
66
67#[cfg(feature = "irc11")]
69unsafe impl<M: AnyUFrameMeta + ?Sized> Objective for DmaCoherentInnerOwner<M> {
70
71}
72
73pub ghost struct DmaCoherentInnerInvariant;
74
75pub type DmaCoherentInnerAtomic<M> = AtomicDataWithOwner<
76 DmaCoherentInner<M>,
77 DmaCoherentInnerInvariant,
78>;
79
80impl<M: AnyUFrameMeta + ?Sized> Inv for DmaCoherent<M> {
81 open spec fn inv(self) -> bool {
82 true
83 }
84}
85
86impl<M: AnyUFrameMeta + ?Sized> Inv for DmaCoherentInner<M> {
87 open spec fn inv(self) -> bool {
88 &&& self.segment.inv()
89 &&& is_valid_daddr(self.start_daddr)
90 }
91}
92
93impl<M: AnyUFrameMeta + ?Sized> Inv for DmaCoherentInnerOwner<M> {
94 open spec fn inv(self) -> bool {
95 true
96 }
97}
98
99impl<M: AnyUFrameMeta + ?Sized> SimpleResourceInvariant<
100 DmaCoherentInner<M>,
101> for DmaCoherentInnerInvariant {
102 type Resource = DmaCoherentInnerOwner<M>;
103
104 open spec fn inv(value: DmaCoherentInner<M>, resource: DmaCoherentInnerOwner<M>) -> bool {
105 &&& resource.inv()
106 &&& value.inv()
107 }
108}
109
110#[verus_verify]
111impl<M: AnyUFrameMeta + ?Sized + OwnerOf> DmaCoherent<M> {
112 #[verus_spec(r =>
117 requires
118 segment.inv(),
119 ensures
120 r matches Ok(r) ==> r.inner.wf(),
121 )]
122 pub fn map(segment: Segment<M>, is_cache_coherent: bool) -> Result<Self, DmaError> {
123 let start_paddr = segment.start_paddr();
124 let frame_count = segment.size() / PAGE_SIZE;
125
126 if !check_and_insert_dma_mapping(start_paddr, frame_count) {
127 return Err(DmaError::AlreadyMapped);
128 }let start_daddr = match dma_type() {
143 DmaType::Direct => {
144 start_paddr as Daddr
154 },
155 DmaType::Iommu => {
156 for i in 0..frame_count
157 invariant
158 segment.inv(),
159 start_paddr == segment.start_paddr(),
160 frame_count == segment.size() / PAGE_SIZE,
161 {
162 assume(start_paddr + i * PAGE_SIZE <= usize::MAX);
163 let _paddr = start_paddr + i * PAGE_SIZE;
164 }
171
172 start_paddr as Daddr
173 },
174 };
175
176 let tracked inner_owner = DmaCoherentInnerOwner { _marker: PhantomData::<M> };
177
178 let inner = RwArc::new(
179 AtomicDataWithOwner::new(
180 DmaCoherentInner { segment, start_daddr, is_cache_coherent },
181 Tracked(inner_owner),
182 Ghost(DmaCoherentInnerInvariant),
183 ),
184 );
185
186 Ok(DmaCoherent { inner })
187 }
188
189 #[inline(always)]
191 #[verus_spec(r =>
192 requires
193 this@.inv(),
194 returns
195 this@.data.start_daddr,
196 )]
197 fn daddr_inner(this: &RwLockReadGuard<DmaCoherentInnerAtomic<M>, PreemptDisabled>) -> Daddr {
198 this.start_daddr
199 }
200
201 #[inline(always)]
203 #[verus_spec(r =>
204 requires
205 this@.inv(),
206 returns
207 this@.data.segment.start_paddr(),
208 )]
209 fn paddr_inner(this: &RwLockReadGuard<DmaCoherentInnerAtomic<M>, PreemptDisabled>) -> Paddr {
210 this.segment.start_paddr()
211 }
212
213 #[inline(always)]
215 #[verus_spec(r =>
216 requires
217 this@.inv(),
218 returns
219 this@.data.segment.size() / PAGE_SIZE,
220 )]
221 fn nframes_inner(this: &RwLockReadGuard<DmaCoherentInnerAtomic<M>, PreemptDisabled>) -> usize {
222 this.segment.size() / PAGE_SIZE
223 }
224
225 #[inline(always)]
227 #[verus_spec(r =>
228 requires
229 this@.inv(),
230 returns
231 this@.data.segment.size(),
232 )]
233 fn nbytes_inner(this: &RwLockReadGuard<DmaCoherentInnerAtomic<M>, PreemptDisabled>) -> usize {
234 this.segment.size()
235 }
236
237 #[inline(always)]
239 #[verus_spec(r =>
240 with
241 -> reader_owner: Tracked<VmIoOwner>,
242 requires
243 this@.inv(),
244 ensures
245 r.inv(),
246 reader_owner@.inv(),
247 reader_owner@.is_kernel,
248 reader_owner@.has_read_view(),
249 reader_owner@.read_view_initialized(),
250 r.wf(reader_owner@),
251 KERNEL_BASE_VADDR <= r.cursor.range@.start,
252 r.cursor.range@.end <= KERNEL_END_VADDR,
253 )]
254 fn reader_inner<'a>(
255 this: RwLockReadGuard<'a, DmaCoherentInnerAtomic<M>, PreemptDisabled>,
256 ) -> VmReader<'a, Infallible> {
257 proof_decl! {
258 let ghost id: nat;
259 let tracked mut reader_owner: VmIoOwner;
260 }
261 proof {
262 lemma_max_paddr_range();
263 lemma_paddr_to_vaddr_properties(this@.data.segment.start_paddr());
264 }
265
266 let vaddr = paddr_to_vaddr(this.segment.start_paddr());
267 let len = this.segment.size();
268 let ghost range = vaddr..(vaddr + len) as usize;
269 let ptr = VirtPtr { vaddr, range: Ghost(range) };
270 proof {
271 assert(KERNEL_BASE_VADDR > 0) by (compute_only);
272 assert(vaddr > 0);
273 assert(VMALLOC_BASE_VADDR <= KERNEL_END_VADDR) by (compute_only);
274 assert(ptr.inv());
275 }
276
277 let reader = unsafe {
278 proof_with!(Ghost(id) => Tracked(reader_owner));
279 VmReader::from_kernel_space(ptr, len)
280 };
281 proof_with!(|= Tracked(reader_owner));
282 reader
283 }
284
285 #[inline(always)]
287 #[verus_spec(r =>
288 with
289 -> writer_owner: Tracked<VmIoOwner>,
290 requires
291 this@.inv(),
292 ensures
293 r.inv(),
294 writer_owner@.inv(),
295 writer_owner@.is_kernel,
296 writer_owner@.has_write_view(),
297 r.wf(writer_owner@),
298 KERNEL_BASE_VADDR <= r.cursor.range@.start,
299 r.cursor.range@.end <= KERNEL_END_VADDR,
300 )]
301 fn writer_inner<'a>(
302 this: RwLockReadGuard<'a, DmaCoherentInnerAtomic<M>, PreemptDisabled>,
303 ) -> VmWriter<'a, Infallible> {
304 proof_decl! {
305 let ghost id: nat;
306 let tracked mut writer_owner: VmIoOwner;
307 }
308 proof {
309 lemma_max_paddr_range();
310 lemma_paddr_to_vaddr_properties(this@.data.segment.start_paddr());
311 }
312
313 let vaddr = paddr_to_vaddr(this.segment.start_paddr());
314 let len = this.segment.size();
315 let ghost range = vaddr..(vaddr + len) as usize;
316 let ptr = VirtPtr { vaddr, range: Ghost(range) };
317 proof {
318 assert(KERNEL_BASE_VADDR > 0) by (compute_only);
319 assert(vaddr > 0);
320 assert(VMALLOC_BASE_VADDR <= KERNEL_END_VADDR) by (compute_only);
321 assert(ptr.inv());
322 }
323
324 let writer = unsafe {
325 proof_with!(Ghost(id), Tracked(false) => Tracked(writer_owner));
326 VmWriter::from_kernel_space(ptr, len)
327 };
328 proof_with!(|= Tracked(writer_owner));
329 writer
330 }
331}
332
333impl<M: AnyUFrameMeta + ?Sized> DmaCoherent<M> {
334 #[inline(always)]
336 #[verifier::external_body]
337 #[verus_spec(r =>
338 ensures
339 self.inner.wf(),
340 r@.inv(),
341 )]
342 pub fn read_inner(&self) -> RwLockReadGuard<'_, DmaCoherentInnerAtomic<M>, PreemptDisabled> {
343 self.inner.read()
344 }
345}
346
347#[verus_verify]
348impl<M: AnyUFrameMeta + ?Sized + OwnerOf> DmaCoherent<M> {
349 #[inline(always)]
351 #[verus_spec(r =>
352 requires
353 self.inner.wf(),
354 ensures
355 self.inner.wf(),
356 )]
357 pub fn daddr(&self) -> Daddr {
358 let inner = self.read_inner();
359 Self::daddr_inner(&inner)
360 }
361
362 #[inline(always)]
364 #[verus_spec(r =>
365 requires
366 self.inner.wf(),
367 ensures
368 self.inner.wf(),
369 )]
370 pub fn paddr(&self) -> Paddr {
371 let inner = self.read_inner();
372 Self::paddr_inner(&inner)
373 }
374
375 #[inline(always)]
377 #[verus_spec(r =>
378 requires
379 self.inner.wf(),
380 ensures
381 self.inner.wf(),
382 )]
383 pub fn nframes(&self) -> usize {
384 let inner = self.read_inner();
385 Self::nframes_inner(&inner)
386 }
387
388 #[inline(always)]
390 #[verus_spec(r =>
391 requires
392 self.inner.wf(),
393 ensures
394 self.inner.wf(),
395 )]
396 pub fn nbytes(&self) -> usize {
397 let inner = self.read_inner();
398 Self::nbytes_inner(&inner)
399 }
400
401 #[inline(always)]
403 #[verus_spec(r =>
404 with
405 -> reader_perm: Tracked<VmIoOwner>,
406 requires
407 self.inner.wf(),
408 ensures
409 self.inner.wf(),
410 r.inv(),
411 reader_perm@.inv(),
412 reader_perm@.is_kernel,
413 reader_perm@.has_read_view(),
414 r.wf(reader_perm@),
415 KERNEL_BASE_VADDR <= r.cursor.range@.start,
416 r.cursor.range@.end <= KERNEL_END_VADDR,
417 )]
418 pub fn reader<'a>(&'a self) -> VmReader<'a, Infallible> {
419 let inner = self.read_inner();
420 proof_with!(=> Tracked(reader_perm));
421 let reader = Self::reader_inner(inner);
422 proof_with!(|= Tracked(reader_perm));
423 VmReader {
424 ghost_id: reader.ghost_id,
425 cursor: reader.cursor,
426 end: reader.end,
427 phantom: PhantomData,
428 }
429 }
430
431 #[inline(always)]
433 #[verus_spec(r =>
434 with
435 -> writer_perm: Tracked<VmIoOwner>,
436 requires
437 self.inner.wf(),
438 ensures
439 self.inner.wf(),
440 r.inv(),
441 writer_perm@.inv(),
442 writer_perm@.is_kernel,
443 writer_perm@.has_write_view(),
444 r.wf(writer_perm@),
445 )]
446 pub fn writer<'a>(&'a self) -> VmWriter<'a, Infallible> {
447 let inner = self.read_inner();
448 proof_with!(=> Tracked(writer_perm));
449 let writer = Self::writer_inner(inner);
450 proof_with!(|= Tracked(writer_perm));
451 VmWriter {
452 ghost_id: writer.ghost_id,
453 cursor: writer.cursor,
454 end: writer.end,
455 phantom: PhantomData,
456 }
457 }
458}
459
460#[verus_verify]
461impl<M: AnyUFrameMeta + ?Sized> Clone for DmaCoherent<M> {
462 #[verus_spec(
463 returns
464 self,
465 )]
466 fn clone(&self) -> Self {
467 DmaCoherent { inner: self.inner.clone() }
468 }
469}
470
471impl<M: AnyUFrameMeta + ?Sized + OwnerOf> HasDaddr for DmaCoherent<M> {
472 #[inline]
473 fn daddr(&self) -> Daddr {
474 let inner = self.read_inner();
475 Self::daddr_inner(&inner)
476 }
477}
478
479impl<M: AnyUFrameMeta + ?Sized + OwnerOf> HasPaddr for DmaCoherent<M> {
480 #[inline]
481 fn paddr(&self) -> Paddr {
482 let inner = self.read_inner();
483 Self::paddr_inner(&inner)
484 }
485}
486
487impl<M: AnyUFrameMeta + ?Sized + Send + Sync + OwnerOf> VmIo<
538 DmaCoherentVmIoOwner<M>,
539> for DmaCoherent<M> {
540 closed spec fn obeys_vmio_spec() -> bool {
541 true
542 }
543
544 closed spec fn obeys_vmio_read_requires() -> bool {
545 true
546 }
547
548 closed spec fn obeys_vmio_write_requires() -> bool {
549 true
550 }
551
552 closed spec fn obeys_vmio_read_spec() -> bool {
553 false
554 }
555
556 closed spec fn obeys_vmio_write_spec() -> bool {
557 false
558 }
559
560 open spec fn read_requires(
561 self,
562 offset: usize,
563 writer: VmWriter<'_>,
564 writer_own: VmIoOwner,
565 owner: DmaCoherentVmIoOwner<M>,
566 ) -> bool {
567 &&& self.inv()
568 &&& self.inner.wf()
569 &&& writer.inv()
570 &&& writer_own.inv()
571 &&& writer.wf(writer_own)
572 &&& writer_own.mem_view matches Some(VmIoMemView::WriteView(_))
573 &&& writer.cursor.vaddr > 0
574 &&& (writer.cursor.range@.end <= KERNEL_BASE_VADDR || KERNEL_END_VADDR
575 <= writer.cursor.range@.start)
576 }
577
578 open spec fn write_requires(
579 self,
580 offset: usize,
581 reader: VmReader<'_>,
582 reader_own: VmIoOwner,
583 owner: DmaCoherentVmIoOwner<M>,
584 ) -> bool {
585 &&& self.inv()
586 &&& self.inner.wf()
587 &&& reader.inv()
588 &&& reader_own.inv()
589 &&& reader.wf(reader_own)
590 &&& reader_own.mem_view matches Some(VmIoMemView::ReadView(_))
591 &&& reader_own.read_view_initialized()
592 &&& reader.cursor.vaddr > 0
593 &&& (reader.cursor.range@.end <= KERNEL_BASE_VADDR || KERNEL_END_VADDR
594 <= reader.cursor.range@.start)
595 }
596
597 open spec fn read_spec(
598 self,
599 offset: usize,
600 old_writer: VmWriter<'_>,
601 new_writer: VmWriter<'_>,
602 old_writer_own: VmIoOwner,
603 new_writer_own: VmIoOwner,
604 old_owner: DmaCoherentVmIoOwner<M>,
605 new_owner: DmaCoherentVmIoOwner<M>,
606 r: core::result::Result<(), Error>,
607 ) -> bool {
608 &&& self.inv()
609 &&& new_owner == old_owner
610 &&& new_writer.inv()
611 &&& new_writer_own.inv()
612 &&& new_writer.wf(new_writer_own)
613 &&& match r {
614 Ok(_) => {
615 &&& new_writer.avail_spec() == 0
616 &&& new_writer.cursor.vaddr == old_writer.cursor.vaddr + old_writer.avail_spec()
617 &&& new_writer_own.range.start == old_writer_own.range.start
618 + old_writer.avail_spec()
619 },
620 Err(_) => {
621 &&& new_writer == old_writer
622 &&& new_writer_own == old_writer_own
623 },
624 }
625 }
626
627 open spec fn write_spec(
628 self,
629 offset: usize,
630 old_reader: VmReader<'_>,
631 new_reader: VmReader<'_>,
632 old_reader_own: VmIoOwner,
633 new_reader_own: VmIoOwner,
634 old_owner: DmaCoherentVmIoOwner<M>,
635 new_owner: DmaCoherentVmIoOwner<M>,
636 r: core::result::Result<(), Error>,
637 ) -> bool {
638 &&& self.inv()
639 &&& new_owner == old_owner
640 &&& new_reader.inv()
641 &&& new_reader_own.inv()
642 &&& new_reader.wf(new_reader_own)
643 &&& match r {
644 Ok(_) => {
645 &&& new_reader.remain_spec() == 0
646 &&& new_reader.cursor.vaddr == old_reader.cursor.vaddr + old_reader.remain_spec()
647 &&& new_reader_own.range.start == old_reader_own.range.start
648 + old_reader.remain_spec()
649 },
650 Err(_) => {
651 &&& new_reader == old_reader
652 &&& new_reader_own == old_reader_own
653 },
654 }
655 }
656
657 fn read(
658 &self,
659 offset: usize,
660 writer: &mut VmWriter<'_>,
661 Tracked(writer_own): Tracked<&mut VmIoOwner>,
662 Tracked(owner): Tracked<&mut DmaCoherentVmIoOwner<M>>,
663 ) -> core::result::Result<(), Error> {
664 proof {
665 assert(Self::obeys_vmio_spec());
666 assert(Self::obeys_vmio_read_requires());
667 assert(!Self::obeys_vmio_read_spec());
668 assert(Self::read_requires(*self, offset, *writer, *writer_own, *owner));
669 }
670 proof_decl! {
671 let tracked mut reader_own;
672 }
673 let mut reader = {
674 let inner = self.read_inner();
675 #[verus_spec(with => Tracked(reader_own))]
676 Self::reader_inner(inner)
677 };
678
679 let remain_before_skip = reader.remain();
680 if offset > remain_before_skip {
681 return Err(Error::InvalidArgs);
682 }
683 let remain = remain_before_skip - offset;
684 let len = writer.avail();
685 if remain < len {
686 return Err(Error::InvalidArgs);
687 }
688 if len == 0 {
689 proof {
690 assert(writer.avail_spec() == 0);
691 }
692 return Ok(());
693 }
694 let reader = reader.skip(offset);
695 proof {
696 reader_own.advance(offset);
697 assert(reader_own.inv());
698 assert(writer_own.inv());
699 assert(reader_own.mem_view matches Some(VmIoMemView::ReadView(_)));
700 assert(reader_own.read_view_initialized());
701 assert(writer_own.mem_view matches Some(VmIoMemView::WriteView(_)));
702 assert(reader.remain_spec() >= len);
703 }
704
705 let copied = match reader.read_fallible(writer) {
706 Ok(copied) => copied,
707 Err((err, copied)) => {
708 writer.cursor = writer.cursor.sub(copied);
709 return Err(err);
710 },
711 };
712
713 proof {
714 reader_own.advance(copied);
715 writer_own.advance(copied);
716 assert(*owner == *old(owner));
717 }
718
719 Ok(())
720 }
721
722 fn write(
723 &self,
724 offset: usize,
725 reader: &mut VmReader<'_>,
726 Tracked(reader_own): Tracked<&mut VmIoOwner>,
727 Tracked(owner): Tracked<&mut DmaCoherentVmIoOwner<M>>,
728 ) -> core::result::Result<(), Error> {
729 proof {
730 assert(Self::obeys_vmio_spec());
731 assert(Self::obeys_vmio_write_requires());
732 assert(!Self::obeys_vmio_write_spec());
733 assert(Self::write_requires(*self, offset, *reader, *reader_own, *owner));
734 }
735 proof_decl! {
736 let tracked mut writer_own;
737 }
738 let mut writer = {
739 let inner = self.read_inner();
740 #[verus_spec(with => Tracked(writer_own))]
741 Self::writer_inner(inner)
742 };
743
744 let avail_before_skip = writer.avail();
745 if offset > avail_before_skip {
746 return Err(Error::InvalidArgs);
747 }
748 let avail = avail_before_skip - offset;
749 let len = reader.remain();
750 if avail < len {
751 return Err(Error::InvalidArgs);
752 }
753 if len == 0 {
754 proof {
755 assert(reader.remain_spec() == 0);
756 }
757 return Ok(());
758 }
759 let writer = writer.skip(offset);
760 proof {
761 writer_own.advance(offset);
762
763 assert(writer.avail_spec() >= len);
764 }
765 let copied = match writer.write_fallible(reader) {
766 Ok(copied) => copied,
767 Err((err, copied)) => {
768 reader.cursor = reader.cursor.sub(copied);
769 return Err(err);
770 },
771 };
772 proof {
773 writer_own.advance(copied);
774 reader_own.advance(copied);
775 assert(*owner == *old(owner));
776 }
777
778 Ok(())
779 }
780}
781
782impl<M: AnyUFrameMeta + ?Sized + Send + Sync + OwnerOf> VmIoOnce for DmaCoherent<M> {
783 closed spec fn obeys_vmio_once_read_requires() -> bool {
784 true
785 }
786
787 closed spec fn obeys_vmio_once_write_requires() -> bool {
788 true
789 }
790
791 closed spec fn obeys_vmio_once_read_ensures() -> bool {
792 true
793 }
794
795 closed spec fn obeys_vmio_once_write_ensures() -> bool {
796 true
797 }
798
799 #[verifier::external_body]
800 fn read_once<T: PodOnce>(&self, offset: usize) -> core::result::Result<T, Error> {
801 self.reader().skip(offset).read_once()
802 }
803
804 #[verifier::external_body]
805 fn write_once<T: PodOnce>(&self, offset: usize, new_val: &T) -> core::result::Result<
806 (),
807 Error,
808 > {
809 self.writer().skip(offset).write_once(new_val)
810 }
811}
812
813}