From nobody Tue Apr 7 04:14:07 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 528B03CF69E for ; Mon, 16 Mar 2026 17:13:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1773681186; cv=none; b=hRN1XajuEZI5vAmcUsmzvIvC5ZOC8J0r2WQLProU2vnNNG/03ePHRYV4AOOHWZUuba9uPlNz/+Xz46RTmqy6Pd326ezPnoGxA3U7shfE2bxSbfI75yrcST1sd+16UTBwDQ/lQkQnTL1/rd0pB5tOT+rOHKfVviCtCdJHNk3lxnc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1773681186; c=relaxed/simple; bh=Bw8djJRJ7prVMkfCnAMP/NyuYr3GVODQM8vnvmX/FSc=; h=Date:Message-ID:From:To:Cc:Subject:References:MIME-Version: Content-Type; b=i5pQ8HoTIpWT5i7qMCrjZ6JpHyjMdSafSGO842inhr/5Zf1QrWP08ifXU9RD6TD7BkMKRfwHuqwbRAnflW0UgD4WhdPF8BhR6VG7LQh9OBgfPXlWBbCPBRAC7P1/gyDEknHg2qiHul0HmRmoLHA5MItzvEr3Hb/PDvP8efFa6GE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=d7ISQ0IT; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="d7ISQ0IT" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 14B44C19425; Mon, 16 Mar 2026 17:13:04 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1773681186; bh=Bw8djJRJ7prVMkfCnAMP/NyuYr3GVODQM8vnvmX/FSc=; h=Date:From:To:Cc:Subject:References:From; b=d7ISQ0ITIUd1tYpc6razMoJ4wsNrS9jOJmlF0/eufnfUl77v0hrvx3IXXFYLBVrgS xO0qzM3loLcLvGXHhqCT0YDhAbU3hu46jj+Nkm1ImOAnQUFcyqOxEBQs5GSxv2Qajt t0N+U2ZMxngypkzGSVOX10dz74QEXjlY0i3xz/OqXQo0sR6gyLDh2RsctkrSuFnXaI 3h1l8YoViMKORjKfH3muAW+GhRX3AXFWxVWWWwJHzMB19+RnwHKlsdtgCWWJRMEuYQ NiJdF2zHrzHxCqb7F8pgrrKuDPvWGkpJzm+PIXSFBG5IfS5P+BSJlbc44DzS+wcQVP CD7giUJQ2eLIQ== Date: Mon, 16 Mar 2026 18:13:02 +0100 Message-ID: <20260316164951.073076616@kernel.org> User-Agent: quilt/0.68 From: Thomas Gleixner To: LKML Cc: Mathieu Desnoyers , =?UTF-8?q?Andr=C3=A9=20Almeida?= , Sebastian Andrzej Siewior , Carlos O'Donell , Peter Zijlstra , Florian Weimer , Rich Felker , Torvald Riegel , Darren Hart , Ingo Molnar , Davidlohr Bueso , Arnd Bergmann , "Liam R . Howlett" Subject: [patch 2/8] futex: Move futex related mm_struct data into a struct References: <20260316162316.356674433@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Having all these members in mm_struct along with the required #ifdeffery is annoying, does not allow efficient initializing of the data with memset() and makes extending it tedious. Move it into a data structure and fix up all usage sites. Signed-off-by: Thomas Gleixner Reviewed-by: Mathieu Desnoyers --- include/linux/futex_types.h | 22 +++++++ include/linux/mm_types.h | 11 --- kernel/futex/core.c | 123 ++++++++++++++++++++-------------------= ----- 3 files changed, 80 insertions(+), 76 deletions(-) --- a/include/linux/futex_types.h +++ b/include/linux/futex_types.h @@ -31,4 +31,26 @@ struct futex_ctrl { struct futex_ctrl { }; #endif /* !CONFIG_FUTEX */ =20 +/** + * struct futex_mm_data - Futex related per MM data + * @phash_lock: Mutex to protect the private hash operations + * @phash: RCU managed pointer to the private hash + * @phash_new: Pointer to a newly allocated private hash + * @phash_batches: Batch state for RCU synchronization + * @phash_rcu: RCU head for call_rcu() + * @phash_atomic: Aggregate value for @phash_ref + * @phash_ref: Per CPU reference counter for a private hash + */ +struct futex_mm_data { +#ifdef CONFIG_FUTEX_PRIVATE_HASH + struct mutex phash_lock; + struct futex_private_hash __rcu *phash; + struct futex_private_hash *phash_new; + unsigned long phash_batches; + struct rcu_head phash_rcu; + atomic_long_t phash_atomic; + unsigned int __percpu *phash_ref; +#endif +}; + #endif /* _LINUX_FUTEX_TYPES_H */ --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -1221,16 +1221,7 @@ struct mm_struct { */ seqcount_t mm_lock_seq; #endif -#ifdef CONFIG_FUTEX_PRIVATE_HASH - struct mutex futex_hash_lock; - struct futex_private_hash __rcu *futex_phash; - struct futex_private_hash *futex_phash_new; - /* futex-ref */ - unsigned long futex_batches; - struct rcu_head futex_rcu; - atomic_long_t futex_atomic; - unsigned int __percpu *futex_ref; -#endif + struct futex_mm_data futex; =20 unsigned long hiwater_rss; /* High-watermark of RSS usage */ unsigned long hiwater_vm; /* High-water virtual memory usage */ --- a/kernel/futex/core.c +++ b/kernel/futex/core.c @@ -188,13 +188,13 @@ static struct futex_hash_bucket * return NULL; =20 if (!fph) - fph =3D rcu_dereference(key->private.mm->futex_phash); + fph =3D rcu_dereference(key->private.mm->futex.phash); if (!fph || !fph->hash_mask) return NULL; =20 - hash =3D jhash2((void *)&key->private.address, - sizeof(key->private.address) / 4, + hash =3D jhash2((void *)&key->private.address, sizeof(key->private.addres= s) / 4, key->both.offset); + return &fph->queues[hash & fph->hash_mask]; } =20 @@ -238,13 +238,12 @@ static bool __futex_pivot_hash(struct mm { struct futex_private_hash *fph; =20 - WARN_ON_ONCE(mm->futex_phash_new); + WARN_ON_ONCE(mm->futex.phash_new); =20 - fph =3D rcu_dereference_protected(mm->futex_phash, - lockdep_is_held(&mm->futex_hash_lock)); + fph =3D rcu_dereference_protected(mm->futex.phash, lockdep_is_held(&mm->f= utex.phash_lock)); if (fph) { if (!futex_ref_is_dead(fph)) { - mm->futex_phash_new =3D new; + mm->futex.phash_new =3D new; return false; } =20 @@ -252,8 +251,8 @@ static bool __futex_pivot_hash(struct mm } new->state =3D FR_PERCPU; scoped_guard(rcu) { - mm->futex_batches =3D get_state_synchronize_rcu(); - rcu_assign_pointer(mm->futex_phash, new); + mm->futex.phash_batches =3D get_state_synchronize_rcu(); + rcu_assign_pointer(mm->futex.phash, new); } kvfree_rcu(fph, rcu); return true; @@ -261,12 +260,12 @@ static bool __futex_pivot_hash(struct mm =20 static void futex_pivot_hash(struct mm_struct *mm) { - scoped_guard(mutex, &mm->futex_hash_lock) { + scoped_guard(mutex, &mm->futex.phash_lock) { struct futex_private_hash *fph; =20 - fph =3D mm->futex_phash_new; + fph =3D mm->futex.phash_new; if (fph) { - mm->futex_phash_new =3D NULL; + mm->futex.phash_new =3D NULL; __futex_pivot_hash(mm, fph); } } @@ -289,7 +288,7 @@ struct futex_private_hash *futex_private scoped_guard(rcu) { struct futex_private_hash *fph; =20 - fph =3D rcu_dereference(mm->futex_phash); + fph =3D rcu_dereference(mm->futex.phash); if (!fph) return NULL; =20 @@ -412,8 +411,7 @@ static int futex_mpol(struct mm_struct * * private hash) is returned if existing. Otherwise a hash bucket from the * global hash is returned. */ -static struct futex_hash_bucket * -__futex_hash(union futex_key *key, struct futex_private_hash *fph) +static struct futex_hash_bucket *__futex_hash(union futex_key *key, struct= futex_private_hash *fph) { int node =3D key->both.node; u32 hash; @@ -426,8 +424,7 @@ static struct futex_hash_bucket * return hb; } =20 - hash =3D jhash2((u32 *)key, - offsetof(typeof(*key), both.offset) / sizeof(u32), + hash =3D jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / sizeof(= u32), key->both.offset); =20 if (node =3D=3D FUTEX_NO_NODE) { @@ -442,8 +439,7 @@ static struct futex_hash_bucket * */ node =3D (hash >> futex_hashshift) % nr_node_ids; if (!node_possible(node)) { - node =3D find_next_bit_wrap(node_possible_map.bits, - nr_node_ids, node); + node =3D find_next_bit_wrap(node_possible_map.bits, nr_node_ids, node); } } =20 @@ -460,9 +456,8 @@ static struct futex_hash_bucket * * Return: Initialized hrtimer_sleeper structure or NULL if no timeout * value given */ -struct hrtimer_sleeper * -futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout, - int flags, u64 range_ns) +struct hrtimer_sleeper *futex_setup_timer(ktime_t *time, struct hrtimer_sl= eeper *timeout, + int flags, u64 range_ns) { if (!time) return NULL; @@ -1551,17 +1546,17 @@ static void __futex_ref_atomic_begin(str * otherwise it would be impossible for it to have reported success * from futex_ref_is_dead(). */ - WARN_ON_ONCE(atomic_long_read(&mm->futex_atomic) !=3D 0); + WARN_ON_ONCE(atomic_long_read(&mm->futex.phash_atomic) !=3D 0); =20 /* * Set the atomic to the bias value such that futex_ref_{get,put}() * will never observe 0. Will be fixed up in __futex_ref_atomic_end() * when folding in the percpu count. */ - atomic_long_set(&mm->futex_atomic, LONG_MAX); + atomic_long_set(&mm->futex.phash_atomic, LONG_MAX); smp_store_release(&fph->state, FR_ATOMIC); =20 - call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu); + call_rcu_hurry(&mm->futex.phash_rcu, futex_ref_rcu); } =20 static void __futex_ref_atomic_end(struct futex_private_hash *fph) @@ -1582,7 +1577,7 @@ static void __futex_ref_atomic_end(struc * Therefore the per-cpu counter is now stable, sum and reset. */ for_each_possible_cpu(cpu) { - unsigned int *ptr =3D per_cpu_ptr(mm->futex_ref, cpu); + unsigned int *ptr =3D per_cpu_ptr(mm->futex.phash_ref, cpu); count +=3D *ptr; *ptr =3D 0; } @@ -1590,7 +1585,7 @@ static void __futex_ref_atomic_end(struc /* * Re-init for the next cycle. */ - this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */ + this_cpu_inc(*mm->futex.phash_ref); /* 0 -> 1 */ =20 /* * Add actual count, subtract bias and initial refcount. @@ -1598,7 +1593,7 @@ static void __futex_ref_atomic_end(struc * The moment this atomic operation happens, futex_ref_is_dead() can * become true. */ - ret =3D atomic_long_add_return(count - LONG_MAX - 1, &mm->futex_atomic); + ret =3D atomic_long_add_return(count - LONG_MAX - 1, &mm->futex.phash_ato= mic); if (!ret) wake_up_var(mm); =20 @@ -1608,8 +1603,8 @@ static void __futex_ref_atomic_end(struc =20 static void futex_ref_rcu(struct rcu_head *head) { - struct mm_struct *mm =3D container_of(head, struct mm_struct, futex_rcu); - struct futex_private_hash *fph =3D rcu_dereference_raw(mm->futex_phash); + struct mm_struct *mm =3D container_of(head, struct mm_struct, futex.phash= _rcu); + struct futex_private_hash *fph =3D rcu_dereference_raw(mm->futex.phash); =20 if (fph->state =3D=3D FR_PERCPU) { /* @@ -1638,7 +1633,7 @@ static void futex_ref_drop(struct futex_ /* * Can only transition the current fph; */ - WARN_ON_ONCE(rcu_dereference_raw(mm->futex_phash) !=3D fph); + WARN_ON_ONCE(rcu_dereference_raw(mm->futex.phash) !=3D fph); /* * We enqueue at least one RCU callback. Ensure mm stays if the task * exits before the transition is completed. @@ -1650,8 +1645,8 @@ static void futex_ref_drop(struct futex_ * * futex_hash() __futex_pivot_hash() * guard(rcu); guard(mm->futex_hash_lock); - * fph =3D mm->futex_phash; - * rcu_assign_pointer(&mm->futex_phash, new); + * fph =3D mm->futex.phash; + * rcu_assign_pointer(&mm->futex.phash, new); * futex_hash_allocate() * futex_ref_drop() * fph->state =3D FR_ATOMIC; @@ -1666,7 +1661,7 @@ static void futex_ref_drop(struct futex_ * There must be at least one full grace-period between publishing a * new fph and trying to replace it. */ - if (poll_state_synchronize_rcu(mm->futex_batches)) { + if (poll_state_synchronize_rcu(mm->futex.phash_batches)) { /* * There was a grace-period, we can begin now. */ @@ -1674,7 +1669,7 @@ static void futex_ref_drop(struct futex_ return; } =20 - call_rcu_hurry(&mm->futex_rcu, futex_ref_rcu); + call_rcu_hurry(&mm->futex.phash_rcu, futex_ref_rcu); } =20 static bool futex_ref_get(struct futex_private_hash *fph) @@ -1684,11 +1679,11 @@ static bool futex_ref_get(struct futex_p guard(preempt)(); =20 if (READ_ONCE(fph->state) =3D=3D FR_PERCPU) { - __this_cpu_inc(*mm->futex_ref); + __this_cpu_inc(*mm->futex.phash_ref); return true; } =20 - return atomic_long_inc_not_zero(&mm->futex_atomic); + return atomic_long_inc_not_zero(&mm->futex.phash_atomic); } =20 static bool futex_ref_put(struct futex_private_hash *fph) @@ -1698,11 +1693,11 @@ static bool futex_ref_put(struct futex_p guard(preempt)(); =20 if (READ_ONCE(fph->state) =3D=3D FR_PERCPU) { - __this_cpu_dec(*mm->futex_ref); + __this_cpu_dec(*mm->futex.phash_ref); return false; } =20 - return atomic_long_dec_and_test(&mm->futex_atomic); + return atomic_long_dec_and_test(&mm->futex.phash_atomic); } =20 static bool futex_ref_is_dead(struct futex_private_hash *fph) @@ -1714,18 +1709,14 @@ static bool futex_ref_is_dead(struct fut if (smp_load_acquire(&fph->state) =3D=3D FR_PERCPU) return false; =20 - return atomic_long_read(&mm->futex_atomic) =3D=3D 0; + return atomic_long_read(&mm->futex.phash_atomic) =3D=3D 0; } =20 int futex_mm_init(struct mm_struct *mm) { - mutex_init(&mm->futex_hash_lock); - RCU_INIT_POINTER(mm->futex_phash, NULL); - mm->futex_phash_new =3D NULL; - /* futex-ref */ - mm->futex_ref =3D NULL; - atomic_long_set(&mm->futex_atomic, 0); - mm->futex_batches =3D get_state_synchronize_rcu(); + memset(&mm->futex, 0, sizeof(mm->futex)); + mutex_init(&mm->futex.phash_lock); + mm->futex.phash_batches =3D get_state_synchronize_rcu(); return 0; } =20 @@ -1733,9 +1724,9 @@ void futex_hash_free(struct mm_struct *m { struct futex_private_hash *fph; =20 - free_percpu(mm->futex_ref); - kvfree(mm->futex_phash_new); - fph =3D rcu_dereference_raw(mm->futex_phash); + free_percpu(mm->futex.phash_ref); + kvfree(mm->futex.phash_new); + fph =3D rcu_dereference_raw(mm->futex.phash); if (fph) kvfree(fph); } @@ -1746,10 +1737,10 @@ static bool futex_pivot_pending(struct m =20 guard(rcu)(); =20 - if (!mm->futex_phash_new) + if (!mm->futex.phash_new) return true; =20 - fph =3D rcu_dereference(mm->futex_phash); + fph =3D rcu_dereference(mm->futex.phash); return futex_ref_is_dead(fph); } =20 @@ -1791,7 +1782,7 @@ static int futex_hash_allocate(unsigned * Once we've disabled the global hash there is no way back. */ scoped_guard(rcu) { - fph =3D rcu_dereference(mm->futex_phash); + fph =3D rcu_dereference(mm->futex.phash); if (fph && !fph->hash_mask) { if (custom) return -EBUSY; @@ -1799,15 +1790,15 @@ static int futex_hash_allocate(unsigned } } =20 - if (!mm->futex_ref) { + if (!mm->futex.phash_ref) { /* * This will always be allocated by the first thread and * therefore requires no locking. */ - mm->futex_ref =3D alloc_percpu(unsigned int); - if (!mm->futex_ref) + mm->futex.phash_ref =3D alloc_percpu(unsigned int); + if (!mm->futex.phash_ref) return -ENOMEM; - this_cpu_inc(*mm->futex_ref); /* 0 -> 1 */ + this_cpu_inc(*mm->futex.phash_ref); /* 0 -> 1 */ } =20 fph =3D kvzalloc(struct_size(fph, queues, hash_slots), @@ -1830,14 +1821,14 @@ static int futex_hash_allocate(unsigned wait_var_event(mm, futex_pivot_pending(mm)); } =20 - scoped_guard(mutex, &mm->futex_hash_lock) { + scoped_guard(mutex, &mm->futex.phash_lock) { struct futex_private_hash *free __free(kvfree) =3D NULL; struct futex_private_hash *cur, *new; =20 - cur =3D rcu_dereference_protected(mm->futex_phash, - lockdep_is_held(&mm->futex_hash_lock)); - new =3D mm->futex_phash_new; - mm->futex_phash_new =3D NULL; + cur =3D rcu_dereference_protected(mm->futex.phash, + lockdep_is_held(&mm->futex.phash_lock)); + new =3D mm->futex.phash_new; + mm->futex.phash_new =3D NULL; =20 if (fph) { if (cur && !cur->hash_mask) { @@ -1847,7 +1838,7 @@ static int futex_hash_allocate(unsigned * the second one returns here. */ free =3D fph; - mm->futex_phash_new =3D new; + mm->futex.phash_new =3D new; return -EBUSY; } if (cur && !new) { @@ -1877,7 +1868,7 @@ static int futex_hash_allocate(unsigned =20 if (new) { /* - * Will set mm->futex_phash_new on failure; + * Will set mm->futex.phash_new on failure; * futex_private_hash_get() will try again. */ if (!__futex_pivot_hash(mm, new) && custom) @@ -1900,7 +1891,7 @@ int futex_hash_allocate_default(void) get_nr_threads(current), num_online_cpus()); =20 - fph =3D rcu_dereference(current->mm->futex_phash); + fph =3D rcu_dereference(current->mm->futex.phash); if (fph) { if (fph->custom) return 0; @@ -1927,7 +1918,7 @@ static int futex_hash_get_slots(void) struct futex_private_hash *fph; =20 guard(rcu)(); - fph =3D rcu_dereference(current->mm->futex_phash); + fph =3D rcu_dereference(current->mm->futex.phash); if (fph && fph->hash_mask) return fph->hash_mask + 1; return 0;