Skip to content

Commit 0e7baeb

Browse files
taiki-etaiki-e
committed
Use MaybeUninit registers
1 parent e1edd61 commit 0e7baeb

14 files changed

Lines changed: 1041 additions & 1603 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,8 @@ jobs:
111111
target: i686-unknown-linux-gnu
112112
- rust: nightly
113113
target: hexagon-unknown-linux-musl
114+
- rust: nightly-2023-08-23 # The last nightly version that doesn't support MaybeUninit registers.
115+
target: hexagon-unknown-linux-musl
114116
- rust: '1.72' # inline asm for loongarch has been stabilized in Rust 1.72
115117
target: loongarch64-unknown-linux-gnu
116118
- rust: stable
@@ -163,6 +165,8 @@ jobs:
163165
target: powerpc64le-unknown-linux-gnu
164166
- rust: nightly
165167
target: riscv32gc-unknown-linux-gnu
168+
- rust: nightly-2023-08-23 # The last nightly version that doesn't support MaybeUninit registers.
169+
target: riscv32gc-unknown-linux-gnu
166170
- rust: '1.59'
167171
target: riscv64gc-unknown-linux-gnu
168172
- rust: stable
@@ -292,6 +296,8 @@ jobs:
292296
- '1.59'
293297
- stable
294298
- beta
299+
- nightly-2023-08-23 # The last nightly version that doesn't support MaybeUninit registers.
300+
- nightly-2023-08-24 # The oldest nightly version that supports MaybeUninit registers: https://github.com/rust-lang/rust/pull/114790
295301
- nightly
296302
runs-on: ubuntu-latest
297303
timeout-minutes: 60

‎src/arch/aarch64.rs‎

Lines changed: 130 additions & 229 deletions
Large diffs are not rendered by default.

‎src/arch/arm.rs‎

Lines changed: 150 additions & 270 deletions
Large diffs are not rendered by default.

‎src/arch/arm_linux.rs‎

Lines changed: 43 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
// - ARMv4 and ARMv5 Differences
77
// https://developer.arm.com/documentation/ddi0406/cb/Appendixes/ARMv4-and-ARMv5-Differences?lang=en
88
//
9-
// Generated asm:
9+
// Generated asm: TODO: update
1010
// - armv5te https://godbolt.org/z/r61s7cnG8
1111
// - armv4t https://godbolt.org/z/xrxfKx1rc
1212

@@ -19,7 +19,10 @@ use core::{
1919
sync::atomic::Ordering,
2020
};
2121

22-
use crate::raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap};
22+
use crate::{
23+
raw::{AtomicCompareExchange, AtomicLoad, AtomicStore, AtomicSwap},
24+
utils::MaybeUninit64,
25+
};
2326

2427
type XSize = usize;
2528

@@ -61,36 +64,29 @@ macro_rules! atomic_load_store {
6164
order: Ordering,
6265
) -> MaybeUninit<Self> {
6366
debug_assert!(src as usize % mem::size_of::<$int_type>() == 0);
64-
let mut out: MaybeUninit<Self> = MaybeUninit::uninit();
65-
let out_ptr = out.as_mut_ptr();
67+
let mut out: MaybeUninit<Self>;
6668

6769
// SAFETY: the caller must uphold the safety contract.
6870
unsafe {
6971
match order {
7072
Ordering::Relaxed => {
7173
asm!(
72-
// (atomic) load from src to tmp
73-
concat!("ldr", $asm_suffix, " {tmp}, [{src}]"),
74-
// store tmp to out
75-
concat!("str", $asm_suffix, " {tmp}, [{out}]"),
74+
// (atomic) load from src to out
75+
concat!("ldr", $asm_suffix, " {out}, [{src}]"),
7676
src = in(reg) src,
77-
out = inout(reg) out_ptr => _,
78-
tmp = lateout(reg) _,
77+
out = lateout(reg) out,
7978
options(nostack, preserves_flags),
8079
);
8180
}
8281
// Acquire and SeqCst loads are equivalent.
8382
Ordering::Acquire | Ordering::SeqCst => {
8483
debug_assert!(kuser_helper_version() >= 3);
8584
asm!(
86-
// (atomic) load from src to tmp
87-
concat!("ldr", $asm_suffix, " {tmp}, [{src}]"),
85+
// (atomic) load from src to out
86+
concat!("ldr", $asm_suffix, " {out}, [{src}]"),
8887
blx!("{kuser_memory_barrier}"), // acquire fence
89-
// store tmp to out
90-
concat!("str", $asm_suffix, " {tmp}, [{out}]"),
9188
src = in(reg) src,
92-
out = inout(reg) out_ptr => _,
93-
tmp = lateout(reg) _,
89+
out = lateout(reg) out,
9490
kuser_memory_barrier = inout(reg) KUSER_MEMORY_BARRIER => _,
9591
out("lr") _,
9692
options(nostack, preserves_flags),
@@ -110,24 +106,20 @@ macro_rules! atomic_load_store {
110106
order: Ordering,
111107
) {
112108
debug_assert!(dst as usize % mem::size_of::<$int_type>() == 0);
113-
let val = val.as_ptr();
114109

115110
// SAFETY: the caller must uphold the safety contract.
116111
unsafe {
117112
macro_rules! atomic_store_release {
118113
($acquire:expr) => {{
119114
debug_assert!(kuser_helper_version() >= 3);
120115
asm!(
121-
// load from val to tmp
122-
concat!("ldr", $asm_suffix, " {tmp}, [{val}]"),
123-
// (atomic) store tmp to dst
116+
// (atomic) store val to dst
124117
blx!("{kuser_memory_barrier}"), // release fence
125-
concat!("str", $asm_suffix, " {tmp}, [{dst}]"),
118+
concat!("str", $asm_suffix, " {val}, [{dst}]"),
126119
$acquire, // acquire fence
127-
dst = inout(reg) dst => _,
120+
dst = in(reg) dst,
128121
val = in(reg) val,
129-
tmp = lateout(reg) _,
130-
kuser_memory_barrier = inout(reg) KUSER_MEMORY_BARRIER => _,
122+
kuser_memory_barrier = in(reg) KUSER_MEMORY_BARRIER,
131123
out("lr") _,
132124
options(nostack, preserves_flags),
133125
)
@@ -136,13 +128,10 @@ macro_rules! atomic_load_store {
136128
match order {
137129
Ordering::Relaxed => {
138130
asm!(
139-
// load from val to tmp
140-
concat!("ldr", $asm_suffix, " {tmp}, [{val}]"),
141-
// (atomic) store tmp to dst
142-
concat!("str", $asm_suffix, " {tmp}, [{dst}]"),
143-
dst = inout(reg) dst => _,
131+
// (atomic) store val to dst
132+
concat!("str", $asm_suffix, " {val}, [{dst}]"),
133+
dst = in(reg) dst,
144134
val = in(reg) val,
145-
tmp = lateout(reg) _,
146135
options(nostack, preserves_flags),
147136
);
148137
}
@@ -168,26 +157,21 @@ macro_rules! atomic {
168157
) -> MaybeUninit<Self> {
169158
debug_assert!(dst as usize % mem::size_of::<$int_type>() == 0);
170159
debug_assert!(kuser_helper_version() >= 2);
171-
let mut out: MaybeUninit<Self> = MaybeUninit::uninit();
172-
let out_ptr = out.as_mut_ptr();
173-
let val = val.as_ptr();
160+
let mut out: MaybeUninit<Self>;
174161

175162
// SAFETY: the caller must uphold the safety contract.
176163
unsafe {
177164
asm!(
178-
"ldr r1, [r1]", // new_val
179165
"2:",
180166
"ldr r0, [r2]", // old_val
181-
"mov {out_tmp}, r0",
167+
"mov {out}, r0",
182168
blx!("{kuser_cmpxchg}"),
183169
"cmp r0, #0",
184170
"bne 2b",
185-
"str {out_tmp}, [{out}]",
186-
out = in(reg) out_ptr,
187-
out_tmp = out(reg) _,
171+
out = out(reg) out,
188172
kuser_cmpxchg = in(reg) KUSER_CMPXCHG,
189173
out("r0") _,
190-
inout("r1") val => _,
174+
in("r1") val, // new_val
191175
in("r2") dst, // ptr
192176
out("r3") _,
193177
out("ip") _,
@@ -210,20 +194,15 @@ macro_rules! atomic {
210194
) -> (MaybeUninit<Self>, bool) {
211195
debug_assert!(dst as usize % mem::size_of::<$int_type>() == 0);
212196
debug_assert!(kuser_helper_version() >= 2);
213-
let mut out: MaybeUninit<Self> = MaybeUninit::uninit();
214-
let out_ptr = out.as_mut_ptr();
215-
let old = old.as_ptr();
216-
let new = new.as_ptr();
197+
let mut out: MaybeUninit<Self>;
217198

218199
// SAFETY: the caller must uphold the safety contract.
219200
unsafe {
220201
let mut r: i32;
221202
asm!(
222-
"ldr {old}, [{old}]",
223-
"ldr {new}, [{new}]",
224203
"2:",
225204
"ldr r0, [r2]", // old_val
226-
"mov {out_tmp}, r0",
205+
"mov {out}, r0",
227206
"cmp r0, {old}",
228207
"bne 3f",
229208
"mov r1, {new}", // new_val
@@ -239,11 +218,9 @@ macro_rules! atomic {
239218
"bne 2b",
240219
"mov r0, #1",
241220
"4:",
242-
"str {out_tmp}, [{out}]",
243-
old = inout(reg) old => _,
244-
new = inout(reg) new => _,
245-
out = in(reg) out_ptr,
246-
out_tmp = out(reg) _,
221+
old = in(reg) old,
222+
new = in(reg) new,
223+
out = out(reg) out,
247224
kuser_cmpxchg = in(reg) KUSER_CMPXCHG,
248225
out("r0") r,
249226
out("r1") _,
@@ -276,34 +253,29 @@ macro_rules! atomic_sub_word {
276253
debug_assert!(dst as usize % mem::size_of::<$int_type>() == 0);
277254
debug_assert!(kuser_helper_version() >= 2);
278255
let (aligned_ptr, shift, mask) = partword::create_mask_values(dst);
279-
let mut out: MaybeUninit<Self> = MaybeUninit::uninit();
280-
let out_ptr = out.as_mut_ptr();
281-
let val = val.as_ptr();
256+
let mut out: MaybeUninit<Self>;
282257

283258
// SAFETY: the caller must uphold the safety contract.
284259
unsafe {
285260
asm!(
286-
concat!("ldr", $asm_suffix, " {val}, [{val}]"),
287261
"lsl {mask}, {mask}, {shift}",
288262
"lsl {val}, {val}, {shift}",
289263
"and {val}, {val}, {mask}",
290264
"mvn {inv_mask}, {mask}",
291265
"2:",
292266
"ldr r0, [r2]", // old_val
293-
"mov {out_tmp}, r0",
267+
"mov {out}, r0",
294268
"and r1, r0, {inv_mask}",
295269
"orr r1, r1, {val}", // new_val
296270
blx!("{kuser_cmpxchg}"),
297271
"cmp r0, #0",
298272
"bne 2b",
299-
"lsr {out_tmp}, {out_tmp}, {shift}",
300-
concat!("str", $asm_suffix, " {out_tmp}, [{out}]"),
301-
val = inout(reg) val => _,
302-
out = in(reg) out_ptr,
273+
"lsr {out}, {out}, {shift}",
274+
val = inout(reg) crate::utils::zero_extend(val) => _,
275+
out = out(reg) out,
303276
shift = in(reg) shift,
304277
mask = inout(reg) mask => _,
305278
inv_mask = out(reg) _,
306-
out_tmp = out(reg) _,
307279
kuser_cmpxchg = in(reg) KUSER_CMPXCHG,
308280
out("r0") _,
309281
out("r1") _,
@@ -330,17 +302,12 @@ macro_rules! atomic_sub_word {
330302
debug_assert!(dst as usize % mem::size_of::<$int_type>() == 0);
331303
debug_assert!(kuser_helper_version() >= 2);
332304
let (aligned_ptr, shift, mask) = partword::create_mask_values(dst);
333-
let mut out: MaybeUninit<Self> = MaybeUninit::uninit();
334-
let out_ptr = out.as_mut_ptr();
335-
let old = old.as_ptr();
336-
let new = new.as_ptr();
305+
let mut out: MaybeUninit<Self>;
337306

338307
// SAFETY: the caller must uphold the safety contract.
339308
unsafe {
340309
let mut r: i32;
341310
asm!(
342-
concat!("ldr", $asm_suffix, " {old}, [{old}]"),
343-
concat!("ldr", $asm_suffix, " {new}, [{new}]"),
344311
"lsl {mask}, {mask}, {shift}",
345312
"lsl {old}, {old}, {shift}",
346313
"lsl {new}, {new}, {shift}",
@@ -350,8 +317,8 @@ macro_rules! atomic_sub_word {
350317
// "mvn {inv_mask}, {mask}",
351318
"2:",
352319
"ldr r0, [r2]", // old_val
353-
"and {out_tmp}, r0, {mask}",
354-
"cmp {out_tmp}, {old}",
320+
"and {out}, r0, {mask}",
321+
"cmp {out}, {old}",
355322
"bne 3f",
356323
"mvn r1, {mask}",
357324
"and r1, r0, r1",
@@ -368,14 +335,12 @@ macro_rules! atomic_sub_word {
368335
"bne 2b",
369336
"mov r0, #1",
370337
"4:",
371-
"lsr {out_tmp}, {out_tmp}, {shift}",
372-
concat!("str", $asm_suffix, " {out_tmp}, [{out}]"),
373-
old = inout(reg) old => _,
374-
new = inout(reg) new => _,
375-
out = in(reg) out_ptr,
338+
"lsr {out}, {out}, {shift}",
339+
old = inout(reg) crate::utils::zero_extend(old) => _,
340+
new = inout(reg) crate::utils::zero_extend(new) => _,
341+
out = out(reg) out,
376342
shift = in(reg) shift,
377343
mask = inout(reg) mask => _,
378-
out_tmp = out(reg) _,
379344
kuser_cmpxchg = in(reg) KUSER_CMPXCHG,
380345
out("r0") r,
381346
out("r1") _,
@@ -529,17 +494,15 @@ macro_rules! atomic64 {
529494
) -> (MaybeUninit<Self>, bool) {
530495
debug_assert!(dst as usize % mem::size_of::<$int_type>() == 0);
531496
assert_has_kuser_cmpxchg64();
497+
let old = MaybeUninit64 { $int_type: old };
532498
let mut out: MaybeUninit<Self> = MaybeUninit::uninit();
533499
let out_ptr = out.as_mut_ptr();
534-
let old = old.as_ptr();
535500
let new = new.as_ptr();
536501

537502
// SAFETY: the caller must uphold the safety contract.
538503
unsafe {
539504
let mut r: i32;
540505
asm!(
541-
"ldr {old_lo}, [{old_hi}]",
542-
"ldr {old_hi}, [{old_hi}, #4]",
543506
"2:",
544507
"ldr r0, [r2]",
545508
"ldr r3, [r2, #4]",
@@ -566,8 +529,8 @@ macro_rules! atomic64 {
566529
"4:",
567530
new = in(reg) new,
568531
out_tmp = in(reg) out_ptr,
569-
old_lo = out(reg) _,
570-
old_hi = inout(reg) old => _,
532+
old_lo = in(reg) old.pair.lo,
533+
old_hi = in(reg) old.pair.hi,
571534
kuser_cmpxchg64 = in(reg) KUSER_CMPXCHG64,
572535
out("r0") r,
573536
out("r1") _,

0 commit comments

Comments
 (0)