kernel/events/core.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-)
I got a report that a task is stuck in perf_event_exit_task() waiting
for global_ctx_data_rwsem. On large systems with lots threads, it'd
have performance issues when it grabs the lock to iterate all threads
in the system to allocate the context data.
And it'd block task exit path which is problematic especially under
memory pressure.
perf_event_open
perf_event_alloc
attach_perf_ctx_data
attach_global_ctx_data
percpu_down_write (global_ctx_data_rwsem)
for_each_process_thread
alloc_task_ctx_data
do_exit
perf_event_exit_task
percpu_down_read (global_ctx_data_rwsem)
It should not hold the global_ctx_data_rwsem on the exit path. Let's
skip allocation for exiting tasks and free the data carefully.
Reported-by: Rosalie Fang <rosaliefang@google.com>
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Namhyung Kim <namhyung@kernel.org>
---
kernel/events/core.c | 20 ++++++++++++++++++--
1 file changed, 18 insertions(+), 2 deletions(-)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 376fb07d869b8b50..e87bb43b7bb3dd4b 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -5421,9 +5421,20 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache,
return -ENOMEM;
for (;;) {
- if (try_cmpxchg((struct perf_ctx_data **)&task->perf_ctx_data, &old, cd)) {
+ if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) {
if (old)
perf_free_ctx_data_rcu(old);
+ /*
+ * Above try_cmpxchg() pairs with try_cmpxchg() from
+ * detach_task_ctx_data() such that
+ * if we race with perf_event_exit_task(), we must
+ * observe PF_EXITING.
+ */
+ if (task->flags & PF_EXITING) {
+ /* detach_task_ctx_data() may free it already */
+ if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL))
+ perf_free_ctx_data_rcu(cd);
+ }
return 0;
}
@@ -5469,6 +5480,8 @@ attach_global_ctx_data(struct kmem_cache *ctx_cache)
/* Allocate everything */
scoped_guard (rcu) {
for_each_process_thread(g, p) {
+ if (p->flags & PF_EXITING)
+ continue;
cd = rcu_dereference(p->perf_ctx_data);
if (cd && !cd->global) {
cd->global = 1;
@@ -14562,8 +14575,11 @@ void perf_event_exit_task(struct task_struct *task)
/*
* Detach the perf_ctx_data for the system-wide event.
+ *
+ * Done without holding global_ctx_data_rwsem; typically
+ * attach_global_ctx_data() will skip over this task, but otherwise
+ * attach_task_ctx_data() will observe PF_EXITING.
*/
- guard(percpu_read)(&global_ctx_data_rwsem);
detach_task_ctx_data(task);
}
--
2.52.0.457.g6b5491de43-googOn Mon, Jan 12, 2026 at 08:51:57AM -0800, Namhyung Kim wrote:
> I got a report that a task is stuck in perf_event_exit_task() waiting
> for global_ctx_data_rwsem. On large systems with lots threads, it'd
> have performance issues when it grabs the lock to iterate all threads
> in the system to allocate the context data.
>
> And it'd block task exit path which is problematic especially under
> memory pressure.
>
> perf_event_open
> perf_event_alloc
> attach_perf_ctx_data
> attach_global_ctx_data
> percpu_down_write (global_ctx_data_rwsem)
> for_each_process_thread
> alloc_task_ctx_data
> do_exit
> perf_event_exit_task
> percpu_down_read (global_ctx_data_rwsem)
>
> It should not hold the global_ctx_data_rwsem on the exit path. Let's
> skip allocation for exiting tasks and free the data carefully.
>
> Reported-by: Rosalie Fang <rosaliefang@google.com>
> Suggested-by: Peter Zijlstra <peterz@infradead.org>
> Signed-off-by: Namhyung Kim <namhyung@kernel.org>
> ---
> kernel/events/core.c | 20 ++++++++++++++++++--
> 1 file changed, 18 insertions(+), 2 deletions(-)
>
> diff --git a/kernel/events/core.c b/kernel/events/core.c
> index 376fb07d869b8b50..e87bb43b7bb3dd4b 100644
> --- a/kernel/events/core.c
> +++ b/kernel/events/core.c
> @@ -5421,9 +5421,20 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache,
> return -ENOMEM;
>
> for (;;) {
> - if (try_cmpxchg((struct perf_ctx_data **)&task->perf_ctx_data, &old, cd)) {
> + if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) {
It seems we need to keep this casting to suppress sparse warnings.
Thanks,
Namhyung
> if (old)
> perf_free_ctx_data_rcu(old);
> + /*
> + * Above try_cmpxchg() pairs with try_cmpxchg() from
> + * detach_task_ctx_data() such that
> + * if we race with perf_event_exit_task(), we must
> + * observe PF_EXITING.
> + */
> + if (task->flags & PF_EXITING) {
> + /* detach_task_ctx_data() may free it already */
> + if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL))
> + perf_free_ctx_data_rcu(cd);
> + }
> return 0;
> }
>
> @@ -5469,6 +5480,8 @@ attach_global_ctx_data(struct kmem_cache *ctx_cache)
> /* Allocate everything */
> scoped_guard (rcu) {
> for_each_process_thread(g, p) {
> + if (p->flags & PF_EXITING)
> + continue;
> cd = rcu_dereference(p->perf_ctx_data);
> if (cd && !cd->global) {
> cd->global = 1;
> @@ -14562,8 +14575,11 @@ void perf_event_exit_task(struct task_struct *task)
>
> /*
> * Detach the perf_ctx_data for the system-wide event.
> + *
> + * Done without holding global_ctx_data_rwsem; typically
> + * attach_global_ctx_data() will skip over this task, but otherwise
> + * attach_task_ctx_data() will observe PF_EXITING.
> */
> - guard(percpu_read)(&global_ctx_data_rwsem);
> detach_task_ctx_data(task);
> }
>
> --
> 2.52.0.457.g6b5491de43-goog
>
Hi Namhyung,
kernel test robot noticed the following build warnings:
[auto build test WARNING on perf-tools-next/perf-tools-next]
[also build test WARNING on tip/perf/core perf-tools/perf-tools linus/master v6.19-rc5 next-20260113]
[cannot apply to acme/perf/core]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Namhyung-Kim/perf-core-Fix-slow-perf_event_task_exit-with-LBR-callstacks/20260113-005301
base: https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git perf-tools-next
patch link: https://lore.kernel.org/r/20260112165157.1919624-1-namhyung%40kernel.org
patch subject: [PATCH] perf/core: Fix slow perf_event_task_exit() with LBR callstacks
config: riscv-randconfig-r111-20260113 (https://download.01.org/0day-ci/archive/20260113/202601132200.HPQCccpc-lkp@intel.com/config)
compiler: riscv64-linux-gcc (GCC) 9.5.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260113/202601132200.HPQCccpc-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202601132200.HPQCccpc-lkp@intel.com/
sparse warnings: (new ones prefixed by >>)
kernel/events/core.c:1486:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:1486:15: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:1486:15: sparse: struct perf_event_context *
kernel/events/core.c:1499:28: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:1499:28: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:1499:28: sparse: struct perf_event_context *
kernel/events/core.c:3669:20: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:3669:20: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:3669:20: sparse: struct perf_event_context *
kernel/events/core.c:3673:18: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:3673:18: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:3673:18: sparse: struct perf_event_context *
kernel/events/core.c:3674:23: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:3674:23: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:3674:23: sparse: struct perf_event_context *
kernel/events/core.c:3727:25: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:3727:25: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:3727:25: sparse: struct perf_event_context *
kernel/events/core.c:3728:25: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:3728:25: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:3728:25: sparse: struct perf_event_context *
kernel/events/core.c:4111:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:4111:15: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:4111:15: sparse: struct perf_event_context *
kernel/events/core.c:4519:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:4519:15: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:4519:15: sparse: struct perf_event_context *
kernel/events/core.c:5008:25: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:5008:25: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:5008:25: sparse: struct perf_event_context *
kernel/events/core.c:6651:9: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6651:9: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6651:9: sparse: struct perf_buffer *
kernel/events/core.c:5283:21: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected struct perf_ctx_data [noderef] __rcu **___op @@ got struct perf_ctx_data **__ai_oldp @@
kernel/events/core.c:5283:21: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected struct perf_ctx_data [noderef] __rcu *__new @@ got struct perf_ctx_data *[assigned] cd @@
>> kernel/events/core.c:5283:21: sparse: sparse: cast removes address space '__rcu' of expression
kernel/events/core.c:5294:37: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected struct perf_ctx_data [noderef] __rcu **___op @@ got struct perf_ctx_data **__ai_oldp @@
kernel/events/core.c:5294:37: sparse: sparse: cast removes address space '__rcu' of expression
kernel/events/core.c:6166:24: sparse: sparse: incorrect type in assignment (different base types) @@ expected restricted __poll_t [usertype] events @@ got int @@
kernel/events/core.c:6402:22: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6402:22: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6402:22: sparse: struct perf_buffer *
kernel/events/core.c:6526:14: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6526:14: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6526:14: sparse: struct perf_buffer *
kernel/events/core.c:6559:14: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6559:14: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6559:14: sparse: struct perf_buffer *
kernel/events/core.c:6672:14: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6672:14: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6672:14: sparse: struct perf_buffer *
kernel/events/core.c:6688:14: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6688:14: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6688:14: sparse: struct perf_buffer *
kernel/events/core.c:8667:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:8667:15: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:8667:15: sparse: struct perf_event_context *
kernel/events/core.c:8755:13: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:8755:13: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:8755:13: sparse: struct perf_buffer *
kernel/events/core.c:8859:61: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected struct task_struct *p @@ got struct task_struct [noderef] __rcu *real_parent @@
kernel/events/core.c:8859:61: sparse: expected struct task_struct *p
kernel/events/core.c:8859:61: sparse: got struct task_struct [noderef] __rcu *real_parent
kernel/events/core.c:8861:61: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected struct task_struct *p @@ got struct task_struct [noderef] __rcu *real_parent @@
kernel/events/core.c:8861:61: sparse: expected struct task_struct *p
kernel/events/core.c:8861:61: sparse: got struct task_struct [noderef] __rcu *real_parent
kernel/events/core.c:9597:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:9597:15: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:9597:15: sparse: struct perf_event_context *
kernel/events/core.c:10792:9: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10792:9: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10792:9: sparse: struct swevent_hlist *
kernel/events/core.c:10831:17: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10831:17: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10831:17: sparse: struct swevent_hlist *
kernel/events/core.c:14213:9: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:14213:9: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:14213:9: sparse: struct perf_event_context *
kernel/events/core.c:14733:17: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:14733:17: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:14733:17: sparse: struct swevent_hlist *
kernel/events/core.c:184:9: sparse: sparse: context imbalance in 'perf_ctx_lock' - wrong count at exit
kernel/events/core.c:200:9: sparse: sparse: context imbalance in 'perf_ctx_unlock' - unexpected unlock
kernel/events/core.c: note: in included file (through include/linux/rbtree.h, include/linux/mm_types.h, include/linux/mmzone.h, ...):
include/linux/rcupdate.h:897:25: sparse: sparse: context imbalance in 'perf_lock_task_context' - different lock contexts for basic block
kernel/events/core.c:1533:17: sparse: sparse: context imbalance in 'perf_pin_task_context' - unexpected unlock
kernel/events/core.c:3009:9: sparse: sparse: context imbalance in '__perf_install_in_context' - wrong count at exit
kernel/events/core.c:4982:17: sparse: sparse: context imbalance in 'find_get_context' - unexpected unlock
kernel/events/core.c: note: in included file:
kernel/events/internal.h:204:46: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected void const [noderef] __user *from @@ got void const *src @@
kernel/events/core.c:10645:17: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10645:17: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10645:17: sparse: struct swevent_hlist *
kernel/events/core.c:10665:17: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10665:17: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10665:17: sparse: struct swevent_hlist *
kernel/events/core.c:10781:16: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10781:16: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10781:16: sparse: struct swevent_hlist *
kernel/events/core.c:10781:16: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10781:16: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10781:16: sparse: struct swevent_hlist *
kernel/events/core.c:10781:16: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10781:16: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10781:16: sparse: struct swevent_hlist *
vim +/__rcu +5283 kernel/events/core.c
5271
5272 static int
5273 attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache,
5274 bool global)
5275 {
5276 struct perf_ctx_data *cd, *old = NULL;
5277
5278 cd = alloc_perf_ctx_data(ctx_cache, global);
5279 if (!cd)
5280 return -ENOMEM;
5281
5282 for (;;) {
> 5283 if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) {
5284 if (old)
5285 perf_free_ctx_data_rcu(old);
5286 /*
5287 * Above try_cmpxchg() pairs with try_cmpxchg() from
5288 * detach_task_ctx_data() such that
5289 * if we race with perf_event_exit_task(), we must
5290 * observe PF_EXITING.
5291 */
5292 if (task->flags & PF_EXITING) {
5293 /* detach_task_ctx_data() may free it already */
5294 if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL))
5295 perf_free_ctx_data_rcu(cd);
5296 }
5297 return 0;
5298 }
5299
5300 if (!old) {
5301 /*
5302 * After seeing a dead @old, we raced with
5303 * removal and lost, try again to install @cd.
5304 */
5305 continue;
5306 }
5307
5308 if (refcount_inc_not_zero(&old->refcount)) {
5309 free_perf_ctx_data(cd); /* unused */
5310 return 0;
5311 }
5312
5313 /*
5314 * @old is a dead object, refcount==0 is stable, try and
5315 * replace it with @cd.
5316 */
5317 }
5318 return 0;
5319 }
5320
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
Hi Namhyung,
kernel test robot noticed the following build warnings:
[auto build test WARNING on perf-tools-next/perf-tools-next]
[also build test WARNING on tip/perf/core perf-tools/perf-tools linus/master v6.19-rc5 next-20260109]
[cannot apply to acme/perf/core]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Namhyung-Kim/perf-core-Fix-slow-perf_event_task_exit-with-LBR-callstacks/20260113-005301
base: https://git.kernel.org/pub/scm/linux/kernel/git/perf/perf-tools-next.git perf-tools-next
patch link: https://lore.kernel.org/r/20260112165157.1919624-1-namhyung%40kernel.org
patch subject: [PATCH] perf/core: Fix slow perf_event_task_exit() with LBR callstacks
config: hexagon-randconfig-r122-20260113 (https://download.01.org/0day-ci/archive/20260113/202601130832.k1P5IrSS-lkp@intel.com/config)
compiler: clang version 22.0.0git (https://github.com/llvm/llvm-project 9b8addffa70cee5b2acc5454712d9cf78ce45710)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260113/202601130832.k1P5IrSS-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202601130832.k1P5IrSS-lkp@intel.com/
sparse warnings: (new ones prefixed by >>)
kernel/events/core.c:1486:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:1486:15: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:1486:15: sparse: struct perf_event_context *
kernel/events/core.c:1499:28: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:1499:28: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:1499:28: sparse: struct perf_event_context *
kernel/events/core.c:3669:20: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:3669:20: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:3669:20: sparse: struct perf_event_context *
kernel/events/core.c:3673:18: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:3673:18: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:3673:18: sparse: struct perf_event_context *
kernel/events/core.c:3674:23: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:3674:23: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:3674:23: sparse: struct perf_event_context *
kernel/events/core.c:3727:25: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:3727:25: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:3727:25: sparse: struct perf_event_context *
kernel/events/core.c:3728:25: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:3728:25: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:3728:25: sparse: struct perf_event_context *
kernel/events/core.c:4111:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:4111:15: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:4111:15: sparse: struct perf_event_context *
kernel/events/core.c:4519:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:4519:15: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:4519:15: sparse: struct perf_event_context *
kernel/events/core.c:5008:25: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:5008:25: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:5008:25: sparse: struct perf_event_context *
kernel/events/core.c:6651:9: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6651:9: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6651:9: sparse: struct perf_buffer *
>> kernel/events/core.c:5283:21: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected struct perf_ctx_data [noderef] __rcu **___op @@ got struct perf_ctx_data **__ai_oldp @@
>> kernel/events/core.c:5283:21: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected struct perf_ctx_data [noderef] __rcu *__new @@ got struct perf_ctx_data *[assigned] cd @@
kernel/events/core.c:5294:37: sparse: sparse: incorrect type in initializer (different address spaces) @@ expected struct perf_ctx_data [noderef] __rcu **___op @@ got struct perf_ctx_data **__ai_oldp @@
kernel/events/core.c:6166:24: sparse: sparse: incorrect type in assignment (different base types) @@ expected restricted __poll_t [usertype] events @@ got int @@
kernel/events/core.c:6402:22: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6402:22: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6402:22: sparse: struct perf_buffer *
kernel/events/core.c:6526:14: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6526:14: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6526:14: sparse: struct perf_buffer *
kernel/events/core.c:6559:14: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6559:14: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6559:14: sparse: struct perf_buffer *
kernel/events/core.c:6672:14: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6672:14: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6672:14: sparse: struct perf_buffer *
kernel/events/core.c:6688:14: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:6688:14: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:6688:14: sparse: struct perf_buffer *
kernel/events/core.c:8667:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:8667:15: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:8667:15: sparse: struct perf_event_context *
kernel/events/core.c:8755:13: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:8755:13: sparse: struct perf_buffer [noderef] __rcu *
kernel/events/core.c:8755:13: sparse: struct perf_buffer *
kernel/events/core.c:8859:61: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected struct task_struct *p @@ got struct task_struct [noderef] __rcu *real_parent @@
kernel/events/core.c:8859:61: sparse: expected struct task_struct *p
kernel/events/core.c:8859:61: sparse: got struct task_struct [noderef] __rcu *real_parent
kernel/events/core.c:8861:61: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected struct task_struct *p @@ got struct task_struct [noderef] __rcu *real_parent @@
kernel/events/core.c:8861:61: sparse: expected struct task_struct *p
kernel/events/core.c:8861:61: sparse: got struct task_struct [noderef] __rcu *real_parent
kernel/events/core.c:9597:15: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:9597:15: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:9597:15: sparse: struct perf_event_context *
kernel/events/core.c:10792:9: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10792:9: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10792:9: sparse: struct swevent_hlist *
kernel/events/core.c:10831:17: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10831:17: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10831:17: sparse: struct swevent_hlist *
kernel/events/core.c:11106:23: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:11106:23: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:11106:23: sparse: struct perf_event_context *
kernel/events/core.c:14213:9: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:14213:9: sparse: struct perf_event_context [noderef] __rcu *
kernel/events/core.c:14213:9: sparse: struct perf_event_context *
kernel/events/core.c:14733:17: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:14733:17: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:14733:17: sparse: struct swevent_hlist *
kernel/events/core.c:184:9: sparse: sparse: context imbalance in 'perf_ctx_lock' - wrong count at exit
kernel/events/core.c:200:9: sparse: sparse: context imbalance in 'perf_ctx_unlock' - unexpected unlock
kernel/events/core.c: note: in included file (through include/linux/rbtree.h, include/linux/mm_types.h, include/linux/mmzone.h, ...):
include/linux/rcupdate.h:895:9: sparse: sparse: context imbalance in 'perf_lock_task_context' - different lock contexts for basic block
kernel/events/core.c:1533:17: sparse: sparse: context imbalance in 'perf_pin_task_context' - unexpected unlock
kernel/events/core.c:3009:9: sparse: sparse: context imbalance in '__perf_install_in_context' - wrong count at exit
kernel/events/core.c:4982:17: sparse: sparse: context imbalance in 'find_get_context' - unexpected unlock
kernel/events/core.c: note: in included file:
kernel/events/internal.h:204:46: sparse: sparse: incorrect type in argument 2 (different address spaces) @@ expected void const [noderef] __user *from @@ got void const *src @@
kernel/events/core.c:10645:17: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10645:17: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10645:17: sparse: struct swevent_hlist *
kernel/events/core.c:10665:17: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10665:17: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10665:17: sparse: struct swevent_hlist *
kernel/events/core.c:10781:16: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10781:16: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10781:16: sparse: struct swevent_hlist *
kernel/events/core.c:10781:16: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10781:16: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10781:16: sparse: struct swevent_hlist *
kernel/events/core.c:10781:16: sparse: sparse: incompatible types in comparison expression (different address spaces):
kernel/events/core.c:10781:16: sparse: struct swevent_hlist [noderef] __rcu *
kernel/events/core.c:10781:16: sparse: struct swevent_hlist *
vim +5283 kernel/events/core.c
5271
5272 static int
5273 attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache,
5274 bool global)
5275 {
5276 struct perf_ctx_data *cd, *old = NULL;
5277
5278 cd = alloc_perf_ctx_data(ctx_cache, global);
5279 if (!cd)
5280 return -ENOMEM;
5281
5282 for (;;) {
> 5283 if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) {
5284 if (old)
5285 perf_free_ctx_data_rcu(old);
5286 /*
5287 * Above try_cmpxchg() pairs with try_cmpxchg() from
5288 * detach_task_ctx_data() such that
5289 * if we race with perf_event_exit_task(), we must
5290 * observe PF_EXITING.
5291 */
5292 if (task->flags & PF_EXITING) {
5293 /* detach_task_ctx_data() may free it already */
5294 if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL))
5295 perf_free_ctx_data_rcu(cd);
5296 }
5297 return 0;
5298 }
5299
5300 if (!old) {
5301 /*
5302 * After seeing a dead @old, we raced with
5303 * removal and lost, try again to install @cd.
5304 */
5305 continue;
5306 }
5307
5308 if (refcount_inc_not_zero(&old->refcount)) {
5309 free_perf_ctx_data(cd); /* unused */
5310 return 0;
5311 }
5312
5313 /*
5314 * @old is a dead object, refcount==0 is stable, try and
5315 * replace it with @cd.
5316 */
5317 }
5318 return 0;
5319 }
5320
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
The following commit has been merged into the perf/core branch of tip:
Commit-ID: 4960626f956d63dce57f099016c2ecbe637a8229
Gitweb: https://git.kernel.org/tip/4960626f956d63dce57f099016c2ecbe637a8229
Author: Namhyung Kim <namhyung@kernel.org>
AuthorDate: Mon, 12 Jan 2026 08:51:57 -08:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 15 Jan 2026 10:04:26 +01:00
perf/core: Fix slow perf_event_task_exit() with LBR callstacks
I got a report that a task is stuck in perf_event_exit_task() waiting
for global_ctx_data_rwsem. On large systems with lots threads, it'd
have performance issues when it grabs the lock to iterate all threads
in the system to allocate the context data.
And it'd block task exit path which is problematic especially under
memory pressure.
perf_event_open
perf_event_alloc
attach_perf_ctx_data
attach_global_ctx_data
percpu_down_write (global_ctx_data_rwsem)
for_each_process_thread
alloc_task_ctx_data
do_exit
perf_event_exit_task
percpu_down_read (global_ctx_data_rwsem)
It should not hold the global_ctx_data_rwsem on the exit path. Let's
skip allocation for exiting tasks and free the data carefully.
Reported-by: Rosalie Fang <rosaliefang@google.com>
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260112165157.1919624-1-namhyung@kernel.org
---
kernel/events/core.c | 20 ++++++++++++++++++--
1 file changed, 18 insertions(+), 2 deletions(-)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 101da5c..da013b9 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -5421,9 +5421,20 @@ attach_task_ctx_data(struct task_struct *task, struct kmem_cache *ctx_cache,
return -ENOMEM;
for (;;) {
- if (try_cmpxchg((struct perf_ctx_data **)&task->perf_ctx_data, &old, cd)) {
+ if (try_cmpxchg(&task->perf_ctx_data, &old, cd)) {
if (old)
perf_free_ctx_data_rcu(old);
+ /*
+ * Above try_cmpxchg() pairs with try_cmpxchg() from
+ * detach_task_ctx_data() such that
+ * if we race with perf_event_exit_task(), we must
+ * observe PF_EXITING.
+ */
+ if (task->flags & PF_EXITING) {
+ /* detach_task_ctx_data() may free it already */
+ if (try_cmpxchg(&task->perf_ctx_data, &cd, NULL))
+ perf_free_ctx_data_rcu(cd);
+ }
return 0;
}
@@ -5469,6 +5480,8 @@ again:
/* Allocate everything */
scoped_guard (rcu) {
for_each_process_thread(g, p) {
+ if (p->flags & PF_EXITING)
+ continue;
cd = rcu_dereference(p->perf_ctx_data);
if (cd && !cd->global) {
cd->global = 1;
@@ -14562,8 +14575,11 @@ void perf_event_exit_task(struct task_struct *task)
/*
* Detach the perf_ctx_data for the system-wide event.
+ *
+ * Done without holding global_ctx_data_rwsem; typically
+ * attach_global_ctx_data() will skip over this task, but otherwise
+ * attach_task_ctx_data() will observe PF_EXITING.
*/
- guard(percpu_read)(&global_ctx_data_rwsem);
detach_task_ctx_data(task);
}
© 2016 - 2026 Red Hat, Inc.