[PATCH RESEND -tip] sched/core: use try_cmpxchg() in mm_cid_pcpu_unset()

Uros Bizjak posted 1 patch 3 weeks, 2 days ago
kernel/sched/sched.h | 15 +++++----------
1 file changed, 5 insertions(+), 10 deletions(-)
[PATCH RESEND -tip] sched/core: use try_cmpxchg() in mm_cid_pcpu_unset()
Posted by Uros Bizjak 3 weeks, 2 days ago
Use try_cmpxchg() instead of cmpxchg (*ptr, old, new) == old in
mm_cid_pcpu_unset(). x86 CMPXCHG instruction returns success in
ZF flag, so this change saves a compare after cmpxchg.

Also, try_cmpxchg() implicitly assigns old *ptr value to "old" when
cmpxchg fails, enabling substantial source code simplifications.

The generated asm code improves from:

    d6cd:	eb 18                	jmp    d6e7 <...>
    d6cf:	65 48 8b 0d 00 00 00 	mov    %gs:0x0(%rip),%rcx
    d6d6:	00
    d6d7:	89 d0                	mov    %edx,%eax
    d6d9:	4c 01 c1             	add    %r8,%rcx
    d6dc:	f0 0f b1 71 08       	lock cmpxchg %esi,0x8(%rcx)
    d6e1:	39 c2                	cmp    %eax,%edx
    d6e3:	74 3a                	je     d71f <...>
    d6e5:	89 c2                	mov    %eax,%edx
    d6e7:	83 fa ff             	cmp    $0xffffffff,%edx
    d6ea:	75 e3                	jne    d6cf <...>
    d6ec:	48 c7 83 20 0a 00 00 	movq   $0xffffffffffffffff,0xa20(%rbx)
    d6f3:	ff ff ff ff
    ...
    d71f:	81 e2 ff ff ff 7f    	and    $0x7fffffff,%edx
    d725:	f0 48 0f b3 95 90 05 	lock btr %rdx,0x590(%rbp)
    d72c:	00 00
    d72e:	eb bc                	jmp    d6ec <...>

to a substatially more streamlined:

    d6cd:	83 f9 ff             	cmp    $0xffffffff,%ecx
    d6d0:	74 23                	je     d6f5 <...>
    d6d2:	65 48 8b 15 00 00 00 	mov    %gs:0x0(%rip),%rdx
    d6d9:	00
    d6da:	89 c8                	mov    %ecx,%eax
    d6dc:	4c 01 c2             	add    %r8,%rdx
    d6df:	f0 0f b1 72 08       	lock cmpxchg %esi,0x8(%rdx)
    d6e4:	75 42                	jne    d728 <...>
    d6e6:	81 e1 ff ff ff 7f    	and    $0x7fffffff,%ecx
    d6ec:	f0 48 0f b3 8d 90 05 	lock btr %rcx,0x590(%rbp)
    d6f3:	00 00
    d6f5:	48 c7 83 20 0a 00 00 	movq   $0xffffffffffffffff,0xa20(%rbx)
    d6fc:	ff ff ff ff
    ...
    d728:	89 c1                	mov    %eax,%ecx
    d72a:	eb a1                	jmp    d6cd <...>

No functional change intended.

Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Juri Lelli <juri.lelli@redhat.com>
Cc: Vincent Guittot <vincent.guittot@linaro.org>
Cc: Dietmar Eggemann <dietmar.eggemann@arm.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Ben Segall <bsegall@google.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: Valentin Schneider <vschneid@redhat.com>
---
 kernel/sched/sched.h | 15 +++++----------
 1 file changed, 5 insertions(+), 10 deletions(-)

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index b5367c514c14..d7f2cf458aa5 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3557,21 +3557,16 @@ static inline void mm_cid_put_lazy(struct task_struct *t)
 static inline int mm_cid_pcpu_unset(struct mm_struct *mm)
 {
 	struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
-	int cid, res;
+	int cid;
 
 	lockdep_assert_irqs_disabled();
 	cid = __this_cpu_read(pcpu_cid->cid);
-	for (;;) {
+	do {
 		if (mm_cid_is_unset(cid))
 			return MM_CID_UNSET;
-		/*
-		 * Attempt transition from valid or lazy-put to unset.
-		 */
-		res = cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, cid, MM_CID_UNSET);
-		if (res == cid)
-			break;
-		cid = res;
-	}
+		/* Attempt transition from valid or lazy-put to unset. */
+	} while (!try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET));
+
 	return cid;
 }
 
-- 
2.51.0