ostd/arch/x86/cpu/
local.rs

1// SPDX-License-Identifier: MPL-2.0
2
3//! Architecture dependent CPU-local information utilities.
4
5use x86_64::registers::segmentation::{GS, Segment64};
6
7/// Gets the base address for the CPU local storage by reading the GS base model-specific register.
8pub(crate) fn get_base() -> u64 {
9    GS::read_base().as_u64()
10}
11
12use crate::cpu::local::single_instr::{
13    SingleInstructionAddAssign, SingleInstructionBitAndAssign, SingleInstructionBitOrAssign,
14    SingleInstructionBitXorAssign, SingleInstructionLoad, SingleInstructionStore,
15    SingleInstructionSubAssign,
16};
17
18macro_rules! impl_numeric_single_instruction_for {
19    ($([$typ: ty, $inout_type: ident, $register_format: expr])*) => {$(
20
21        impl SingleInstructionAddAssign<$typ> for $typ {
22            unsafe fn add_assign(offset: *mut Self, val: Self) {
23                // SAFETY:
24                // 1. `gs` points to the CPU-local region (global invariant).
25                // 2. `offset` represents the offset of a CPU-local variable
26                //    (upheld by the caller).
27                // 3. The variable is only accessible in the current CPU, is
28                //    a scalar, and is never borrowed, so it is valid to
29                //    read/write using a single instruction (upheld by the
30                //    caller).
31                unsafe {
32                    core::arch::asm!(
33                        concat!("add gs:[{0}], {1", $register_format, "}"),
34                        in(reg) offset,
35                        in($inout_type) val,
36                        options(nostack),
37                    );
38                }
39            }
40        }
41
42        impl SingleInstructionSubAssign<$typ> for $typ {
43            unsafe fn sub_assign(offset: *mut Self, val: Self) {
44                // SAFETY: Same as `add_assign`.
45                unsafe {
46                    core::arch::asm!(
47                        concat!("sub gs:[{0}], {1", $register_format, "}"),
48                        in(reg) offset,
49                        in($inout_type) val,
50                        options(nostack),
51                    );
52                }
53            }
54        }
55
56        impl SingleInstructionBitAndAssign<$typ> for $typ {
57            unsafe fn bitand_assign(offset: *mut Self, val: Self) {
58                // SAFETY: Same as `add_assign`.
59                unsafe {
60                    core::arch::asm!(
61                        concat!("and gs:[{0}], {1", $register_format, "}"),
62                        in(reg) offset,
63                        in($inout_type) val,
64                        options(nostack),
65                    );
66                }
67            }
68        }
69
70        impl SingleInstructionBitOrAssign<$typ> for $typ {
71            unsafe fn bitor_assign(offset: *mut Self, val: Self) {
72                // SAFETY: Same as `add_assign`.
73                unsafe {
74                    core::arch::asm!(
75                        concat!("or gs:[{0}], {1", $register_format, "}"),
76                        in(reg) offset,
77                        in($inout_type) val,
78                        options(nostack),
79                    );
80                }
81            }
82        }
83
84        impl SingleInstructionBitXorAssign<$typ> for $typ {
85            unsafe fn bitxor_assign(offset: *mut Self, val: Self) {
86                // SAFETY: Same as `add_assign`.
87                unsafe {
88                    core::arch::asm!(
89                        concat!("xor gs:[{0}], {1", $register_format, "}"),
90                        in(reg) offset,
91                        in($inout_type) val,
92                        options(nostack),
93                    );
94                }
95            }
96        }
97
98        impl SingleInstructionLoad for $typ {
99            unsafe fn load(offset: *const Self) -> Self {
100                let val: Self;
101                // SAFETY: Same as `add_assign`.
102                unsafe {
103                    core::arch::asm!(
104                        concat!("mov {0", $register_format, "}, gs:[{1}]"),
105                        out($inout_type) val,
106                        in(reg) offset,
107                        options(nostack, readonly),
108                    );
109                }
110                val
111            }
112        }
113
114        impl SingleInstructionStore for $typ {
115            unsafe fn store(offset: *mut Self, val: Self) {
116                // SAFETY: Same as `add_assign`.
117                unsafe {
118                    core::arch::asm!(
119                        concat!("mov gs:[{0}], {1", $register_format, "}"),
120                        in(reg) offset,
121                        in($inout_type) val,
122                        options(nostack),
123                    );
124                }
125            }
126        }
127
128    )*};
129}
130
131impl_numeric_single_instruction_for!(
132    [u64,   reg,    ":r"]
133    [usize, reg,    ":r"]
134    [u32,   reg,    ":e"]
135    [u16,   reg,    ":x"]
136    [u8,    reg_byte, ""]
137    [i64,   reg,    ":r"]
138    [isize, reg,    ":r"]
139    [i32,   reg,    ":e"]
140    [i16,   reg,    ":x"]
141    [i8,    reg_byte, ""]
142);
143
144macro_rules! impl_generic_single_instruction_for {
145    ($([<$gen_type:ident $(, $more_gen_type:ident)*>, $typ:ty])*) => {$(
146
147        impl<$gen_type $(, $more_gen_type)*> SingleInstructionLoad for $typ {
148            unsafe fn load(offset: *const Self) -> Self {
149                let val: Self;
150                // SAFETY: Same as `add_assign`.
151                unsafe {
152                    core::arch::asm!(
153                        concat!("mov {0}, gs:[{1}]"),
154                        out(reg) val,
155                        in(reg) offset,
156                        options(nostack, readonly),
157                    );
158                }
159                val
160            }
161        }
162
163        impl<$gen_type $(, $more_gen_type)*> SingleInstructionStore for $typ {
164            unsafe fn store(offset: *mut Self, val: Self) {
165                // SAFETY: Same as `add_assign`.
166                unsafe {
167                    core::arch::asm!(
168                        concat!("mov gs:[{0}], {1}"),
169                        in(reg) offset,
170                        in(reg) val,
171                        options(nostack),
172                    );
173                }
174            }
175        }
176    )*}
177}
178
179impl_generic_single_instruction_for!(
180    [<T>, *const T]
181    [<T>, *mut T]
182    [<T, R>, fn(T) -> R]
183);
184
185// In this module, booleans are represented by the least significant bit of a
186// `u8` type. Other bits must be zero. This definition is compatible with the
187// Rust reference: <https://doc.rust-lang.org/reference/types/boolean.html>.
188
189impl SingleInstructionLoad for bool {
190    unsafe fn load(offset: *const Self) -> Self {
191        let val: u8;
192        // SAFETY: Same as `add_assign`.
193        unsafe {
194            core::arch::asm!(
195                "mov {0}, gs:[{1}]",
196                out(reg_byte) val,
197                in(reg) offset,
198                options(nostack, readonly),
199            );
200        }
201        debug_assert!(val == 1 || val == 0);
202        val == 1
203    }
204}
205
206impl SingleInstructionStore for bool {
207    unsafe fn store(offset: *mut Self, val: Self) {
208        let val: u8 = if val { 1 } else { 0 };
209        // SAFETY: Same as `add_assign`.
210        unsafe {
211            core::arch::asm!(
212                "mov gs:[{0}], {1}",
213                in(reg) offset,
214                in(reg_byte) val,
215                options(nostack),
216            );
217        }
218    }
219}
ostd/arch/x86/cpu/local.rs

ostd/arch/x86/cpu/
local.rs
