ostd/
smp.rs

1// SPDX-License-Identifier: MPL-2.0
2
3//! Symmetric Multi-Processing (SMP) support.
4//!
5//! This module provides a way to execute code on other processors via inter-
6//! processor interrupts.
7//!
8//! Callers issue work with [`inter_processor_call`]. Remote calls are queued for
9//! interrupt-context execution and return a [`PendingIpis`] handle that can be
10//! waited on when the caller needs completion.
11
12use alloc::{boxed::Box, collections::VecDeque};
13use core::sync::atomic::{AtomicBool, Ordering};
14
15use spin::Once;
16
17use crate::{
18    arch::{irq::HwCpuId, trap::TrapFrame},
19    cpu::{CpuSet, PinCurrentCpu},
20    cpu_local, irq,
21    sync::SpinLock,
22    util::id_set::Id,
23};
24
25/// Executes a function on other processors.
26///
27/// The provided function `call_fn` will be executed on all target processors
28/// specified by `targets`. It can also be executed on the current processor.
29/// The function should be short and non-blocking, as it will be executed in
30/// interrupt context with interrupts disabled.
31///
32/// The function `call_fn` will be executed asynchronously on the target
33/// processors. However, if called on the current processor, it will be
34/// synchronous.
35///
36/// The returned [`PendingIpis`] can be used to wait until all remote target
37/// processors have handled IPIs for this call.
38///
39/// # Panics
40///
41/// This function will panic if a hardware error occurs while sending an IPI to
42/// a remote processor.
43pub fn inter_processor_call(targets: &CpuSet, call_fn: fn()) -> PendingIpis {
44    let ipi_sender = IPI_SENDER.get().unwrap();
45    ipi_sender.inter_processor_call(targets, call_fn)
46}
47
48/// Pending remote inter-processor calls.
49pub struct PendingIpis {
50    cpus: CpuSet,
51}
52
53impl PendingIpis {
54    pub(crate) fn new_empty() -> Self {
55        Self {
56            cpus: CpuSet::new_empty(),
57        }
58    }
59
60    fn add(&mut self, cpu_id: crate::cpu::CpuId) {
61        self.cpus.add(cpu_id);
62    }
63
64    pub(crate) fn extend(&mut self, other: &Self) {
65        for cpu_id in other.cpus.iter() {
66            self.add(cpu_id);
67        }
68    }
69
70    /// Waits until all pending remote processors have handled their IPIs.
71    ///
72    /// # Panics
73    ///
74    /// This method panics if local IRQs are disabled. Waiting for remote IPIs
75    /// with local IRQs disabled can deadlock if one of the remote processors is
76    /// also waiting for this processor to handle an IPI.
77    pub fn wait(&self) {
78        assert!(
79            crate::arch::irq::is_local_enabled(),
80            "waiting for remote inter-processor calls with IRQs disabled"
81        );
82
83        for cpu_id in self.cpus.iter() {
84            // Wait until there are no pending IPIs on the target CPU.
85            //
86            // Note that if new IPIs arrive to that CPU in the meantime, we
87            // will also wait for them. This is fine because there usually
88            // aren't too many IPIs in common cases.
89            while HAS_PENDING_IPIS.get_on_cpu(cpu_id).load(Ordering::Acquire) {
90                core::hint::spin_loop();
91            }
92        }
93    }
94}
95
96/// A sender that carries necessary information to send inter-processor interrupts.
97///
98/// The purpose of exporting this type is to enable the users to check whether
99/// [`IPI_SENDER`] has been initialized.
100pub(crate) struct IpiSender {
101    hw_cpu_ids: Box<[HwCpuId]>,
102}
103
104/// The [`IpiSender`] singleton.
105pub(crate) static IPI_SENDER: Once<IpiSender> = Once::new();
106
107impl IpiSender {
108    /// Executes a function on other processors.
109    ///
110    /// See [`inter_processor_call`] for details. The purpose of exporting this
111    /// method is to enable callers to check whether [`IPI_SENDER`] has been
112    /// initialized.
113    pub(crate) fn inter_processor_call(&self, targets: &CpuSet, call_fn: fn()) -> PendingIpis {
114        let irq_guard = irq::disable_local();
115        let this_cpu_id = irq_guard.current_cpu();
116
117        let mut call_on_self = false;
118        let mut pending_ipis = PendingIpis::new_empty();
119        for cpu_id in targets.iter() {
120            if cpu_id == this_cpu_id {
121                call_on_self = true;
122                continue;
123            }
124            let mut call_queue = CALL_QUEUES.get_on_cpu(cpu_id).lock();
125            call_queue.push_back(call_fn);
126            // Set the pending flag before dropping the lock to avoid races.
127            HAS_PENDING_IPIS
128                .get_on_cpu(cpu_id)
129                .store(true, Ordering::Release);
130            pending_ipis.add(cpu_id);
131        }
132        for cpu_id in targets.iter() {
133            if cpu_id == this_cpu_id {
134                continue;
135            }
136            let hw_cpu_id = self.hw_cpu_ids[cpu_id.as_usize()];
137            crate::arch::irq::send_ipi(hw_cpu_id, &irq_guard as _)
138                .expect("failed to send inter-processor interrupt");
139        }
140        if call_on_self {
141            // Execute the function synchronously.
142            call_fn();
143        }
144        pending_ipis
145    }
146}
147
148cpu_local! {
149    static CALL_QUEUES: SpinLock<VecDeque<fn()>> = SpinLock::new(VecDeque::new());
150    static HAS_PENDING_IPIS: AtomicBool = AtomicBool::new(false);
151}
152
153/// Handles inter-processor calls.
154///
155/// # Safety
156///
157/// This function must be called from an IRQ handler that can be triggered by
158/// inter-processor interrupts.
159pub(crate) unsafe fn do_inter_processor_call(_trapframe: &TrapFrame) {
160    // No races because we are in IRQs.
161    let this_cpu_id = crate::cpu::CpuId::current_racy();
162
163    let mut queue = CALL_QUEUES.get_on_cpu(this_cpu_id).lock();
164    while let Some(call_fn) = queue.pop_front() {
165        crate::debug!(
166            "Performing inter-processor call to {:#?} on CPU {:#?}",
167            call_fn,
168            this_cpu_id,
169        );
170        call_fn();
171    }
172    // Clear the pending flag before dropping the lock to avoid races.
173    HAS_PENDING_IPIS
174        .get_on_cpu(this_cpu_id)
175        .store(false, Ordering::Release);
176}
177
178pub(super) fn init() {
179    IPI_SENDER.call_once(|| {
180        let hw_cpu_ids = crate::boot::smp::construct_hw_cpu_id_mapping();
181        IpiSender { hw_cpu_ids }
182    });
183}
184
185#[cfg(ktest)]
186mod test {
187    use core::sync::atomic::{AtomicUsize, Ordering};
188
189    use crate::{
190        cpu::{self, PinCurrentCpu},
191        prelude::ktest,
192        task,
193    };
194
195    static IPI_CALL_COUNT: AtomicUsize = AtomicUsize::new(0);
196
197    fn count_ipi_call() {
198        IPI_CALL_COUNT.fetch_add(1, Ordering::Relaxed);
199    }
200
201    #[ktest]
202    fn pending_ipis_waits_for_all_cpus() {
203        let before = IPI_CALL_COUNT.load(Ordering::Relaxed);
204
205        super::inter_processor_call(&cpu::CpuSet::new_full(), count_ipi_call).wait();
206
207        assert_eq!(
208            IPI_CALL_COUNT.load(Ordering::Relaxed) - before,
209            cpu::num_cpus()
210        );
211    }
212
213    #[ktest]
214    fn inter_processor_call_runs_on_current_cpu() {
215        let preempt_guard = task::disable_preempt();
216        let before = IPI_CALL_COUNT.load(Ordering::Relaxed);
217
218        super::inter_processor_call(
219            &cpu::CpuSet::from(preempt_guard.current_cpu()),
220            count_ipi_call,
221        )
222        .wait();
223
224        assert_eq!(IPI_CALL_COUNT.load(Ordering::Relaxed) - before, 1);
225    }
226
227    #[ktest]
228    fn pending_ipis_waits_for_remote_cpu() {
229        if cpu::num_cpus() < 2 {
230            return;
231        }
232
233        let preempt_guard = task::disable_preempt();
234        let target_cpu = cpu::all_cpus()
235            .find(|cpu_id| *cpu_id != preempt_guard.current_cpu())
236            .unwrap();
237        let before = IPI_CALL_COUNT.load(Ordering::Relaxed);
238
239        super::inter_processor_call(&cpu::CpuSet::from(target_cpu), count_ipi_call).wait();
240
241        assert_eq!(IPI_CALL_COUNT.load(Ordering::Relaxed) - before, 1);
242    }
243
244    #[ktest]
245    fn pending_ipis_can_be_extended() {
246        if cpu::num_cpus() < 2 {
247            return;
248        }
249
250        let preempt_guard = task::disable_preempt();
251        let target_cpu = cpu::all_cpus()
252            .find(|cpu_id| *cpu_id != preempt_guard.current_cpu())
253            .unwrap();
254        let target_cpus = cpu::CpuSet::from(target_cpu);
255        let before = IPI_CALL_COUNT.load(Ordering::Relaxed);
256
257        let mut pending_ipis = super::PendingIpis::new_empty();
258        pending_ipis.extend(&super::inter_processor_call(&target_cpus, count_ipi_call));
259        pending_ipis.extend(&super::inter_processor_call(&target_cpus, count_ipi_call));
260        pending_ipis.wait();
261
262        assert_eq!(IPI_CALL_COUNT.load(Ordering::Relaxed) - before, 2);
263    }
264}
ostd/smp.rs

ostd/
smp.rs
