kernel/sync/
barrier.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
// SPDX-License-Identifier: GPL-2.0

//! Memory barriers.
//!
//! These primitives have the same semantics as their C counterparts: and the precise definitions
//! of semantics can be found at [`LKMM`].
//!
//! [`LKMM`]: srctree/tools/memory-model/

#![expect(private_bounds, reason = "sealed implementation")]

/// Memory barrier orderings.
///
/// The semantics of these orderings follows the [`LKMM`] definitions and rules.
///
/// - [`Read`] provides ordering between preceding load operations and succeeding load operations.
/// - [`Write`] provides ordering between preceding store operations and succeeding store
///   operations.
/// - [`Full`] provides ordering between all the preceding memory accesses and succeeding memory
///   accesses.
///
/// [`LKMM`]: srctree/tools/memory-model/
pub mod ordering {
    pub use crate::sync::atomic::ordering::Full;

    /// The annotation type for read-read barrier ordering.
    pub struct Read;

    /// The annotation type for write-write barrier ordering.
    pub struct Write;
}

pub use ordering::{
    Full,
    Read,
    Write, //
};

struct Smp;
struct Dma;

/// A compiler barrier.
///
/// A barrier that prevents compiler from reordering memory accesses across the barrier.
#[inline(always)]
pub(crate) fn barrier() {
    // By default, Rust inline asms are treated as being able to access any memory or flags, hence
    // it suffices as a compiler barrier.
    //
    // SAFETY: An empty asm block.
    unsafe { core::arch::asm!("") };
}

trait MemoryBarrier<Flavour = ()> {
    fn run();
}

macro_rules! define_barrier {
    ($([$flavour:ident])? $ordering:ident, $binding:ident) => {
        impl MemoryBarrier$(<$flavour>)? for $ordering {
            #[inline]
            fn run() {
                // SAFETY: barrier methods are safe to call.
                unsafe { bindings::$binding() };
            }
        }
    };
}

define_barrier!(Full, mb);
define_barrier!(Read, rmb);
define_barrier!(Write, wmb);
define_barrier!([Dma] Full, dma_mb);
define_barrier!([Dma] Read, dma_rmb);
define_barrier!([Dma] Write, dma_wmb);
define_barrier!([Smp] Full, smp_mb);
define_barrier!([Smp] Read, smp_rmb);
define_barrier!([Smp] Write, smp_wmb);

/// Memory barrier.
///
/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
///
/// The specific forms of reordering can be specified using the parameter.
/// - `mb(Read)` provides a read-read barrier.
/// - `mb(Write)` provides a write-write barrier.
/// - `mb(Full)` provides a full barrier.
///
/// # Examples
///
/// ```
/// # use kernel::sync::barrier::*;
/// mb(Read);
/// mb(Write);
/// mb(Full);
/// ```
#[inline]
#[doc(alias = "rmb")]
#[doc(alias = "wmb")]
pub fn mb<T: MemoryBarrier>(_: T) {
    T::run()
}

/// Memory barrier between CPUs.
///
/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
/// Does not prevent re-ordering with respect to other bus-mastering devices.
///
/// See [`mb`] for usage.
#[inline]
#[doc(alias = "smp_rmb")]
#[doc(alias = "smp_wmb")]
pub fn smp_mb<T: MemoryBarrier<Smp>>(_: T) {
    if cfg!(CONFIG_SMP) {
        T::run()
    } else {
        barrier()
    }
}

/// Memory barrier between local CPU and bus-mastering devices.
///
/// A barrier that prevents compiler and CPU from reordering memory accesses across the barrier.
/// Does not prevent re-ordering with respect to other CPUs.
///
/// See [`mb`] for usage.
#[inline]
#[doc(alias = "dma_rmb")]
#[doc(alias = "dma_wmb")]
pub fn dma_mb<T: MemoryBarrier<Dma>>(_: T) {
    T::run()
}