drivers/acpi/osl.c | 2 +- include/linux/mutex.h | 2 +- include/linux/mutex_types.h | 2 +- include/linux/rwsem.h | 8 ++-- include/linux/semaphore.h | 4 +- kernel/locking/mutex-debug.c | 5 +- kernel/locking/mutex.c | 49 +++++++++++--------- kernel/locking/rwsem.c | 89 +++++++++++++++++++++++------------- kernel/locking/semaphore.c | 41 +++++++++++++---- kernel/locking/ww_mutex.h | 25 +++------- 10 files changed, 132 insertions(+), 95 deletions(-)
This series shrinks the mutex, semaphore and rw_semaphore by one pointer by changing the list_head to a pointer to the first waiter. Some sample savings with the Debian config: task_struct 5696 to 5632 bytes (89 to 88 cache lines) mm_struct 1728 to 1600 bytes (27 to 25 cache lines) mm_mm_cid 192 to 128 bytes (3 to 2 cache lines) file 184 to 176 bytes (from 22 objects per 4kB slab to 23) inode 608 to 584 bytes (from 13 objects per 8kB slab to 14) Changes from RFC: - Added patches for semaphore & mutex - Undid inadvertent change to rwsem_del_wake_waiter() - Attempted to follow Peter's preferred coding style Matthew Wilcox (Oracle) (3): rwsem: Remove the list_head from struct rw_semaphore semaphore: Remove the list_head from struct semaphore mutex: Remove the list_head from struct mutex drivers/acpi/osl.c | 2 +- include/linux/mutex.h | 2 +- include/linux/mutex_types.h | 2 +- include/linux/rwsem.h | 8 ++-- include/linux/semaphore.h | 4 +- kernel/locking/mutex-debug.c | 5 +- kernel/locking/mutex.c | 49 +++++++++++--------- kernel/locking/rwsem.c | 89 +++++++++++++++++++++++------------- kernel/locking/semaphore.c | 41 +++++++++++++---- kernel/locking/ww_mutex.h | 25 +++------- 10 files changed, 132 insertions(+), 95 deletions(-) -- 2.47.3
On Thu, Mar 05, 2026 at 07:55:40PM +0000, Matthew Wilcox (Oracle) wrote:
> Matthew Wilcox (Oracle) (3):
> rwsem: Remove the list_head from struct rw_semaphore
> semaphore: Remove the list_head from struct semaphore
> mutex: Remove the list_head from struct mutex
>
> drivers/acpi/osl.c | 2 +-
> include/linux/mutex.h | 2 +-
> include/linux/mutex_types.h | 2 +-
> include/linux/rwsem.h | 8 ++--
> include/linux/semaphore.h | 4 +-
> kernel/locking/mutex-debug.c | 5 +-
> kernel/locking/mutex.c | 49 +++++++++++---------
> kernel/locking/rwsem.c | 89 +++++++++++++++++++++++-------------
> kernel/locking/semaphore.c | 41 +++++++++++++----
> kernel/locking/ww_mutex.h | 25 +++-------
> 10 files changed, 132 insertions(+), 95 deletions(-)
Right, it also completely messes up my context analysis patches, but
that's my own damn fault for not having merged them yet, so I rebased
them on top of this.
I've also added a patch for rwsem since I was there anyway.
I shall push out the entire pile into queue/locking/core for the robots
to chew on.
Thanks!
---
Subject: locking/rwsem: Add context analysis
From: Peter Zijlstra <peterz@infradead.org>
Date: Fri Mar 6 10:43:56 CET 2026
Add compiler context analysis annotations.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
include/linux/rwsem.h | 4 ++--
kernel/locking/Makefile | 1 +
kernel/locking/rwbase_rt.c | 1 +
kernel/locking/rwsem.c | 25 +++++++++++++++++++++++--
4 files changed, 27 insertions(+), 4 deletions(-)
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -57,7 +57,7 @@ context_lock_struct(rw_semaphore) {
struct optimistic_spin_queue osq; /* spinner MCS lock */
#endif
raw_spinlock_t wait_lock;
- struct rwsem_waiter *first_waiter;
+ struct rwsem_waiter *first_waiter __guarded_by(&wait_lock);
#ifdef CONFIG_DEBUG_RWSEMS
void *magic;
#endif
@@ -131,7 +131,7 @@ do { \
*/
static inline bool rwsem_is_contended(struct rw_semaphore *sem)
{
- return sem->first_waiter != NULL;
+ return data_race(sem->first_waiter != NULL);
}
#if defined(CONFIG_DEBUG_RWSEMS) || defined(CONFIG_DETECT_HUNG_TASK_BLOCKER)
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -6,6 +6,7 @@ KCOV_INSTRUMENT := n
CONTEXT_ANALYSIS_mutex.o := y
CONTEXT_ANALYSIS_rtmutex_api.o := y
CONTEXT_ANALYSIS_ww_rt_mutex.o := y
+CONTEXT_ANALYSIS_rwsem.o := y
obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
--- a/kernel/locking/rwbase_rt.c
+++ b/kernel/locking/rwbase_rt.c
@@ -186,6 +186,7 @@ static __always_inline void rwbase_read_
static inline void __rwbase_write_unlock(struct rwbase_rt *rwb, int bias,
unsigned long flags)
+ __releases(&rwb->rtmutex.wait_lock)
{
struct rt_mutex_base *rtm = &rwb->rtmutex;
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -320,9 +320,10 @@ void __init_rwsem(struct rw_semaphore *s
sem->magic = sem;
#endif
atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);
- raw_spin_lock_init(&sem->wait_lock);
- sem->first_waiter = NULL;
atomic_long_set(&sem->owner, 0L);
+ scoped_guard (raw_spinlock_init, &sem->wait_lock) {
+ sem->first_waiter = NULL;
+ }
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
osq_lock_init(&sem->osq);
#endif
@@ -365,6 +366,7 @@ enum rwsem_wake_type {
static inline
bool __rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
+ __must_hold(&sem->wait_lock)
{
if (list_empty(&waiter->list)) {
sem->first_waiter = NULL;
@@ -401,6 +403,7 @@ rwsem_del_waiter(struct rw_semaphore *se
static inline
struct rwsem_waiter *next_waiter(const struct rw_semaphore *sem,
const struct rwsem_waiter *waiter)
+ __must_hold(&sem->wait_lock)
{
struct rwsem_waiter *next = list_first_entry(&waiter->list,
struct rwsem_waiter, list);
@@ -621,6 +624,7 @@ rwsem_del_wake_waiter(struct rw_semaphor
*/
static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
struct rwsem_waiter *waiter)
+ __must_hold(&sem->wait_lock)
{
struct rwsem_waiter *first = sem->first_waiter;
long count, new;
@@ -1558,6 +1562,7 @@ static inline bool is_rwsem_reader_owned
* lock for reading
*/
void __sched down_read(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1567,6 +1572,7 @@ void __sched down_read(struct rw_semapho
EXPORT_SYMBOL(down_read);
int __sched down_read_interruptible(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1581,6 +1587,7 @@ int __sched down_read_interruptible(stru
EXPORT_SYMBOL(down_read_interruptible);
int __sched down_read_killable(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1598,6 +1605,7 @@ EXPORT_SYMBOL(down_read_killable);
* trylock for reading -- returns 1 if successful, 0 if contention
*/
int down_read_trylock(struct rw_semaphore *sem)
+ __no_context_analysis
{
int ret = __down_read_trylock(sem);
@@ -1611,6 +1619,7 @@ EXPORT_SYMBOL(down_read_trylock);
* lock for writing
*/
void __sched down_write(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1622,6 +1631,7 @@ EXPORT_SYMBOL(down_write);
* lock for writing
*/
int __sched down_write_killable(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1640,6 +1650,7 @@ EXPORT_SYMBOL(down_write_killable);
* trylock for writing -- returns 1 if successful, 0 if contention
*/
int down_write_trylock(struct rw_semaphore *sem)
+ __no_context_analysis
{
int ret = __down_write_trylock(sem);
@@ -1654,6 +1665,7 @@ EXPORT_SYMBOL(down_write_trylock);
* release a read lock
*/
void up_read(struct rw_semaphore *sem)
+ __no_context_analysis
{
rwsem_release(&sem->dep_map, _RET_IP_);
__up_read(sem);
@@ -1664,6 +1676,7 @@ EXPORT_SYMBOL(up_read);
* release a write lock
*/
void up_write(struct rw_semaphore *sem)
+ __no_context_analysis
{
rwsem_release(&sem->dep_map, _RET_IP_);
__up_write(sem);
@@ -1674,6 +1687,7 @@ EXPORT_SYMBOL(up_write);
* downgrade write lock to read lock
*/
void downgrade_write(struct rw_semaphore *sem)
+ __no_context_analysis
{
lock_downgrade(&sem->dep_map, _RET_IP_);
__downgrade_write(sem);
@@ -1683,6 +1697,7 @@ EXPORT_SYMBOL(downgrade_write);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
void down_read_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1691,6 +1706,7 @@ void down_read_nested(struct rw_semaphor
EXPORT_SYMBOL(down_read_nested);
int down_read_killable_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1705,6 +1721,7 @@ int down_read_killable_nested(struct rw_
EXPORT_SYMBOL(down_read_killable_nested);
void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
@@ -1713,6 +1730,7 @@ void _down_write_nest_lock(struct rw_sem
EXPORT_SYMBOL(_down_write_nest_lock);
void down_read_non_owner(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
__down_read(sem);
@@ -1727,6 +1745,7 @@ void down_read_non_owner(struct rw_semap
EXPORT_SYMBOL(down_read_non_owner);
void down_write_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1735,6 +1754,7 @@ void down_write_nested(struct rw_semapho
EXPORT_SYMBOL(down_write_nested);
int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1750,6 +1770,7 @@ int __sched down_write_killable_nested(s
EXPORT_SYMBOL(down_write_killable_nested);
void up_read_non_owner(struct rw_semaphore *sem)
+ __no_context_analysis
{
DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);
__up_read(sem);
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 739690915ce1f017223ef4e6f3cc966ccfa3c861
Gitweb: https://git.kernel.org/tip/739690915ce1f017223ef4e6f3cc966ccfa3c861
Author: Peter Zijlstra <peterz@infradead.org>
AuthorDate: Fri, 06 Mar 2026 10:43:56 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sun, 08 Mar 2026 11:06:53 +01:00
locking/rwsem: Add context analysis
Add compiler context analysis annotations.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260306101417.GT1282955@noisy.programming.kicks-ass.net
---
include/linux/rwsem.h | 4 ++--
kernel/locking/Makefile | 1 +
kernel/locking/rwbase_rt.c | 1 +
kernel/locking/rwsem.c | 27 ++++++++++++++++++++++++---
4 files changed, 28 insertions(+), 5 deletions(-)
diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h
index e782953..6a1a7ba 100644
--- a/include/linux/rwsem.h
+++ b/include/linux/rwsem.h
@@ -57,7 +57,7 @@ context_lock_struct(rw_semaphore) {
struct optimistic_spin_queue osq; /* spinner MCS lock */
#endif
raw_spinlock_t wait_lock;
- struct rwsem_waiter *first_waiter;
+ struct rwsem_waiter *first_waiter __guarded_by(&wait_lock);
#ifdef CONFIG_DEBUG_RWSEMS
void *magic;
#endif
@@ -131,7 +131,7 @@ do { \
*/
static inline bool rwsem_is_contended(struct rw_semaphore *sem)
{
- return sem->first_waiter != NULL;
+ return data_race(sem->first_waiter != NULL);
}
#if defined(CONFIG_DEBUG_RWSEMS) || defined(CONFIG_DETECT_HUNG_TASK_BLOCKER)
diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile
index 0c07de7..cee1901 100644
--- a/kernel/locking/Makefile
+++ b/kernel/locking/Makefile
@@ -6,6 +6,7 @@ KCOV_INSTRUMENT := n
CONTEXT_ANALYSIS_mutex.o := y
CONTEXT_ANALYSIS_rtmutex_api.o := y
CONTEXT_ANALYSIS_ww_rt_mutex.o := y
+CONTEXT_ANALYSIS_rwsem.o := y
obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o
diff --git a/kernel/locking/rwbase_rt.c b/kernel/locking/rwbase_rt.c
index 9f4322c..82e078c 100644
--- a/kernel/locking/rwbase_rt.c
+++ b/kernel/locking/rwbase_rt.c
@@ -186,6 +186,7 @@ static __always_inline void rwbase_read_unlock(struct rwbase_rt *rwb,
static inline void __rwbase_write_unlock(struct rwbase_rt *rwb, int bias,
unsigned long flags)
+ __releases(&rwb->rtmutex.wait_lock)
{
struct rt_mutex_base *rtm = &rwb->rtmutex;
diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
index e66f37e..ba4cb74 100644
--- a/kernel/locking/rwsem.c
+++ b/kernel/locking/rwsem.c
@@ -72,7 +72,7 @@
#c, atomic_long_read(&(sem)->count), \
(unsigned long) sem->magic, \
atomic_long_read(&(sem)->owner), (long)current, \
- (sem)->first_waiter ? "" : "not ")) \
+ rwsem_is_contended(sem) ? "" : "not ")) \
debug_locks_off(); \
} while (0)
#else
@@ -320,9 +320,10 @@ void __init_rwsem(struct rw_semaphore *sem, const char *name,
sem->magic = sem;
#endif
atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE);
- raw_spin_lock_init(&sem->wait_lock);
- sem->first_waiter = NULL;
atomic_long_set(&sem->owner, 0L);
+ scoped_guard (raw_spinlock_init, &sem->wait_lock) {
+ sem->first_waiter = NULL;
+ }
#ifdef CONFIG_RWSEM_SPIN_ON_OWNER
osq_lock_init(&sem->osq);
#endif
@@ -365,6 +366,7 @@ enum rwsem_wake_type {
static inline
bool __rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
+ __must_hold(&sem->wait_lock)
{
if (list_empty(&waiter->list)) {
sem->first_waiter = NULL;
@@ -401,6 +403,7 @@ rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter)
static inline
struct rwsem_waiter *next_waiter(const struct rw_semaphore *sem,
const struct rwsem_waiter *waiter)
+ __must_hold(&sem->wait_lock)
{
struct rwsem_waiter *next = list_first_entry(&waiter->list,
struct rwsem_waiter, list);
@@ -621,6 +624,7 @@ rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter,
*/
static inline bool rwsem_try_write_lock(struct rw_semaphore *sem,
struct rwsem_waiter *waiter)
+ __must_hold(&sem->wait_lock)
{
struct rwsem_waiter *first = sem->first_waiter;
long count, new;
@@ -1558,6 +1562,7 @@ static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem)
* lock for reading
*/
void __sched down_read(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1567,6 +1572,7 @@ void __sched down_read(struct rw_semaphore *sem)
EXPORT_SYMBOL(down_read);
int __sched down_read_interruptible(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1581,6 +1587,7 @@ int __sched down_read_interruptible(struct rw_semaphore *sem)
EXPORT_SYMBOL(down_read_interruptible);
int __sched down_read_killable(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1598,6 +1605,7 @@ EXPORT_SYMBOL(down_read_killable);
* trylock for reading -- returns 1 if successful, 0 if contention
*/
int down_read_trylock(struct rw_semaphore *sem)
+ __no_context_analysis
{
int ret = __down_read_trylock(sem);
@@ -1611,6 +1619,7 @@ EXPORT_SYMBOL(down_read_trylock);
* lock for writing
*/
void __sched down_write(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1622,6 +1631,7 @@ EXPORT_SYMBOL(down_write);
* lock for writing
*/
int __sched down_write_killable(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
@@ -1640,6 +1650,7 @@ EXPORT_SYMBOL(down_write_killable);
* trylock for writing -- returns 1 if successful, 0 if contention
*/
int down_write_trylock(struct rw_semaphore *sem)
+ __no_context_analysis
{
int ret = __down_write_trylock(sem);
@@ -1654,6 +1665,7 @@ EXPORT_SYMBOL(down_write_trylock);
* release a read lock
*/
void up_read(struct rw_semaphore *sem)
+ __no_context_analysis
{
rwsem_release(&sem->dep_map, _RET_IP_);
__up_read(sem);
@@ -1664,6 +1676,7 @@ EXPORT_SYMBOL(up_read);
* release a write lock
*/
void up_write(struct rw_semaphore *sem)
+ __no_context_analysis
{
rwsem_release(&sem->dep_map, _RET_IP_);
__up_write(sem);
@@ -1674,6 +1687,7 @@ EXPORT_SYMBOL(up_write);
* downgrade write lock to read lock
*/
void downgrade_write(struct rw_semaphore *sem)
+ __no_context_analysis
{
lock_downgrade(&sem->dep_map, _RET_IP_);
__downgrade_write(sem);
@@ -1683,6 +1697,7 @@ EXPORT_SYMBOL(downgrade_write);
#ifdef CONFIG_DEBUG_LOCK_ALLOC
void down_read_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1691,6 +1706,7 @@ void down_read_nested(struct rw_semaphore *sem, int subclass)
EXPORT_SYMBOL(down_read_nested);
int down_read_killable_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1705,6 +1721,7 @@ int down_read_killable_nested(struct rw_semaphore *sem, int subclass)
EXPORT_SYMBOL(down_read_killable_nested);
void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
@@ -1713,6 +1730,7 @@ void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
EXPORT_SYMBOL(_down_write_nest_lock);
void down_read_non_owner(struct rw_semaphore *sem)
+ __no_context_analysis
{
might_sleep();
__down_read(sem);
@@ -1727,6 +1745,7 @@ void down_read_non_owner(struct rw_semaphore *sem)
EXPORT_SYMBOL(down_read_non_owner);
void down_write_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1735,6 +1754,7 @@ void down_write_nested(struct rw_semaphore *sem, int subclass)
EXPORT_SYMBOL(down_write_nested);
int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
+ __no_context_analysis
{
might_sleep();
rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
@@ -1750,6 +1770,7 @@ int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
EXPORT_SYMBOL(down_write_killable_nested);
void up_read_non_owner(struct rw_semaphore *sem)
+ __no_context_analysis
{
DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem);
__up_read(sem);© 2016 - 2026 Red Hat, Inc.