kernel/sched/sched.h | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-)
Use try_cmpxchg() instead of cmpxchg (*ptr, old, new) == old in
mm_cid_pcpu_unset(). x86 CMPXCHG instruction returns success in
ZF flag, so this change saves a compare after cmpxchg.
Also, try_cmpxchg() implicitly assigns old *ptr value to "old" when
cmpxchg fails, enabling substantial source code simplifications.
The generated asm code improves from:
d6cd: eb 18 jmp d6e7 <...>
d6cf: 65 48 8b 0d 00 00 00 mov %gs:0x0(%rip),%rcx
d6d6: 00
d6d7: 89 d0 mov %edx,%eax
d6d9: 4c 01 c1 add %r8,%rcx
d6dc: f0 0f b1 71 08 lock cmpxchg %esi,0x8(%rcx)
d6e1: 39 c2 cmp %eax,%edx
d6e3: 74 3a je d71f <...>
d6e5: 89 c2 mov %eax,%edx
d6e7: 83 fa ff cmp $0xffffffff,%edx
d6ea: 75 e3 jne d6cf <...>
d6ec: 48 c7 83 20 0a 00 00 movq $0xffffffffffffffff,0xa20(%rbx)
d6f3: ff ff ff ff
...
d71f: 81 e2 ff ff ff 7f and $0x7fffffff,%edx
d725: f0 48 0f b3 95 90 05 lock btr %rdx,0x590(%rbp)
d72c: 00 00
d72e: eb bc jmp d6ec <...>
to a substatially more streamlined:
d6cd: 83 f9 ff cmp $0xffffffff,%ecx
d6d0: 74 23 je d6f5 <...>
d6d2: 65 48 8b 15 00 00 00 mov %gs:0x0(%rip),%rdx
d6d9: 00
d6da: 89 c8 mov %ecx,%eax
d6dc: 4c 01 c2 add %r8,%rdx
d6df: f0 0f b1 72 08 lock cmpxchg %esi,0x8(%rdx)
d6e4: 75 42 jne d728 <...>
d6e6: 81 e1 ff ff ff 7f and $0x7fffffff,%ecx
d6ec: f0 48 0f b3 8d 90 05 lock btr %rcx,0x590(%rbp)
d6f3: 00 00
d6f5: 48 c7 83 20 0a 00 00 movq $0xffffffffffffffff,0xa20(%rbx)
d6fc: ff ff ff ff
...
d728: 89 c1 mov %eax,%ecx
d72a: eb a1 jmp d6cd <...>
No functional change intended.
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Juri Lelli <juri.lelli@redhat.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Ben Segall <bsegall@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Valentin Schneider <vschneid@redhat.com>
---
kernel/sched/sched.h | 15 +++++----------
1 file changed, 5 insertions(+), 10 deletions(-)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index b5367c514c14..d7f2cf458aa5 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3557,21 +3557,16 @@ static inline void mm_cid_put_lazy(struct task_struct *t)
static inline int mm_cid_pcpu_unset(struct mm_struct *mm)
{
struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid, res;
+ int cid;
lockdep_assert_irqs_disabled();
cid = __this_cpu_read(pcpu_cid->cid);
- for (;;) {
+ do {
if (mm_cid_is_unset(cid))
return MM_CID_UNSET;
- /*
- * Attempt transition from valid or lazy-put to unset.
- */
- res = cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, cid, MM_CID_UNSET);
- if (res == cid)
- break;
- cid = res;
- }
+ /* Attempt transition from valid or lazy-put to unset. */
+ } while (!try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET));
+
return cid;
}
--
2.51.0
© 2016 - 2025 Red Hat, Inc.