Unlike sched_skip_vma_numa tracepoint which tracks skipped VMAs, this
tracks the task subjected to cpuset.mems pinning and prints out its
allowed memory node mask.
---
 include/trace/events/sched.h | 31 +++++++++++++++++++++++++++++++
 kernel/sched/fair.c          |  4 +++-
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h
index bfd97cce40a1a..133d9a671734a 100644
--- a/include/trace/events/sched.h
+++ b/include/trace/events/sched.h
@@ -745,6 +745,37 @@ TRACE_EVENT(sched_skip_vma_numa,
 		  __entry->vm_end,
 		  __print_symbolic(__entry->reason, NUMAB_SKIP_REASON))
 );
+
+TRACE_EVENT(sched_skip_cpuset_numa,
+
+	TP_PROTO(struct task_struct *tsk, nodemask_t *mem_allowed_ptr),
+
+	TP_ARGS(tsk, mem_allowed_ptr),
+
+	TP_STRUCT__entry(
+		__array( char,		comm,		TASK_COMM_LEN		)
+		__field( pid_t,		pid					)
+		__field( pid_t,		tgid					)
+		__field( pid_t,		ngid					)
+		__array( unsigned long, mem_allowed, BITS_TO_LONGS(MAX_NUMNODES))
+	),
+
+	TP_fast_assign(
+		memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
+		__entry->pid		 = task_pid_nr(tsk);
+		__entry->tgid		 = task_tgid_nr(tsk);
+		__entry->ngid		 = task_numa_group_id(tsk);
+		memcpy(__entry->mem_allowed, mem_allowed_ptr->bits,
+		       sizeof(__entry->mem_allowed));
+	),
+
+	TP_printk("comm=%s pid=%d tgid=%d ngid=%d mem_node_allowed_mask=%lx",
+		  __entry->comm,
+		  __entry->pid,
+		  __entry->tgid,
+		  __entry->ngid,
+		  __entry->mem_allowed[0])
+);
 #endif /* CONFIG_NUMA_BALANCING */
 
 /*
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 6f405e00c9c7e..a98842a96eda0 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -3333,8 +3333,10 @@ static void task_numa_work(struct callback_head *work)
 	 * Memory is pinned to only one NUMA node via cpuset.mems, naturally
 	 * no page can be migrated.
 	 */
-	if (nodes_weight(cpuset_current_mems_allowed) == 1)
+	if (nodes_weight(cpuset_current_mems_allowed) == 1) {
+		trace_sched_skip_cpuset_numa(current, &cpuset_current_mems_allowed);
 		return;
+	}
 
 	if (!mm->numa_next_scan) {
 		mm->numa_next_scan = now +
-- 
2.43.5

forgot to add Steven Rostedt.

On 3/26/25 17:23, Libo Chen wrote:
> Unlike sched_skip_vma_numa tracepoint which tracks skipped VMAs, this
> tracks the task subjected to cpuset.mems pinning and prints out its
> allowed memory node mask.
> ---
>  include/trace/events/sched.h | 31 +++++++++++++++++++++++++++++++
>  kernel/sched/fair.c          |  4 +++-
>  2 files changed, 34 insertions(+), 1 deletion(-)
> 
> diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h
> index bfd97cce40a1a..133d9a671734a 100644
> --- a/include/trace/events/sched.h
> +++ b/include/trace/events/sched.h
> @@ -745,6 +745,37 @@ TRACE_EVENT(sched_skip_vma_numa,
>  		  __entry->vm_end,
>  		  __print_symbolic(__entry->reason, NUMAB_SKIP_REASON))
>  );
> +
> +TRACE_EVENT(sched_skip_cpuset_numa,
> +
> +	TP_PROTO(struct task_struct *tsk, nodemask_t *mem_allowed_ptr),
> +
> +	TP_ARGS(tsk, mem_allowed_ptr),
> +
> +	TP_STRUCT__entry(
> +		__array( char,		comm,		TASK_COMM_LEN		)
> +		__field( pid_t,		pid					)
> +		__field( pid_t,		tgid					)
> +		__field( pid_t,		ngid					)
> +		__array( unsigned long, mem_allowed, BITS_TO_LONGS(MAX_NUMNODES))
> +	),
> +
> +	TP_fast_assign(
> +		memcpy(__entry->comm, tsk->comm, TASK_COMM_LEN);
> +		__entry->pid		 = task_pid_nr(tsk);
> +		__entry->tgid		 = task_tgid_nr(tsk);
> +		__entry->ngid		 = task_numa_group_id(tsk);
> +		memcpy(__entry->mem_allowed, mem_allowed_ptr->bits,
> +		       sizeof(__entry->mem_allowed));
> +	),
> +
> +	TP_printk("comm=%s pid=%d tgid=%d ngid=%d mem_node_allowed_mask=%lx",


I cannot find a way to print out nodemask_t nicely here with %*pbl.
So I fall back to just raw hex value. Will be grateful if someone
knows a better way to print nodemask nicely in a tracepoint  


> +		  __entry->comm,
> +		  __entry->pid,
> +		  __entry->tgid,
> +		  __entry->ngid,
> +		  __entry->mem_allowed[0])
> +);
>  #endif /* CONFIG_NUMA_BALANCING */
>  
>  /*
> diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
> index 6f405e00c9c7e..a98842a96eda0 100644
> --- a/kernel/sched/fair.c
> +++ b/kernel/sched/fair.c
> @@ -3333,8 +3333,10 @@ static void task_numa_work(struct callback_head *work)
>  	 * Memory is pinned to only one NUMA node via cpuset.mems, naturally
>  	 * no page can be migrated.
>  	 */
> -	if (nodes_weight(cpuset_current_mems_allowed) == 1)
> +	if (nodes_weight(cpuset_current_mems_allowed) == 1) {
> +		trace_sched_skip_cpuset_numa(current, &cpuset_current_mems_allowed);
>  		return;
> +	}
>  
>  	if (!mm->numa_next_scan) {
>  		mm->numa_next_scan = now +

On Wed, 26 Mar 2025 17:40:02 -0700
Libo Chen <libo.chen@oracle.com> wrote:

> > +	TP_printk("comm=%s pid=%d tgid=%d ngid=%d mem_node_allowed_mask=%lx",  
> 
> 
> I cannot find a way to print out nodemask_t nicely here with %*pbl.
> So I fall back to just raw hex value. Will be grateful if someone
> knows a better way to print nodemask nicely in a tracepoint  

Ug.

I guess I need to add support for that. Both in the boot verifier as well
as in libtraceevent.

-- Steve

On 3/26/25 19:06, Steven Rostedt wrote:
> On Wed, 26 Mar 2025 17:40:02 -0700
> Libo Chen <libo.chen@oracle.com> wrote:
> 
>>> +	TP_printk("comm=%s pid=%d tgid=%d ngid=%d mem_node_allowed_mask=%lx",  
>>
>>
>> I cannot find a way to print out nodemask_t nicely here with %*pbl.
>> So I fall back to just raw hex value. Will be grateful if someone
>> knows a better way to print nodemask nicely in a tracepoint  
> 
> Ug.
> 
> I guess I need to add support for that. Both in the boot verifier as well
> as in libtraceevent.
> 

That will be fantastic, I think this is quite useful.

Thanks,
Libo

> -- Steve

On Wed, 26 Mar 2025 21:32:32 -0700
Libo Chen <libo.chen@oracle.com> wrote:

> On 3/26/25 19:06, Steven Rostedt wrote:
> > On Wed, 26 Mar 2025 17:40:02 -0700
> > Libo Chen <libo.chen@oracle.com> wrote:
> >   
> >>> +	TP_printk("comm=%s pid=%d tgid=%d ngid=%d mem_node_allowed_mask=%lx",    
> >>
> >>
> >> I cannot find a way to print out nodemask_t nicely here with %*pbl.
> >> So I fall back to just raw hex value. Will be grateful if someone
> >> knows a better way to print nodemask nicely in a tracepoint    
> > 
> > Ug.
> > 
> > I guess I need to add support for that. Both in the boot verifier as well
> > as in libtraceevent.
> >   
> 
> That will be fantastic, I think this is quite useful.

I'm curious to what was broken. Just the user space side? Or was the kernel
side broken too?

I notice that i2c_write has "[%*phD]" which means it worked, and also
skipped out on the verification tests. This should be fixed.

But trace-cmd and perf will not be able to print that out either.

Now %*pbl works in the kernel (not checked), but probably doesn't work for
trace-cmd.

-- Steve

On 3/27/25 07:03, Steven Rostedt wrote:
> On Wed, 26 Mar 2025 21:32:32 -0700
> Libo Chen <libo.chen@oracle.com> wrote:
> 
>> On 3/26/25 19:06, Steven Rostedt wrote:
>>> On Wed, 26 Mar 2025 17:40:02 -0700
>>> Libo Chen <libo.chen@oracle.com> wrote:
>>>   
>>>>> +	TP_printk("comm=%s pid=%d tgid=%d ngid=%d mem_node_allowed_mask=%lx",    
>>>>
>>>>
>>>> I cannot find a way to print out nodemask_t nicely here with %*pbl.
>>>> So I fall back to just raw hex value. Will be grateful if someone
>>>> knows a better way to print nodemask nicely in a tracepoint    
>>>
>>> Ug.
>>>
>>> I guess I need to add support for that. Both in the boot verifier as well
>>> as in libtraceevent.
>>>   
>>
>> That will be fantastic, I think this is quite useful.
>

Oops just saw this right now, not sure how I missed it.
 
> I'm curious to what was broken. Just the user space side? Or was the kernel
> side broken too?
> 

That's a good point, I originally only tried %*pbl with trace-cmd, but I
tested your patch without trace-cmd. So yeah %*pbl may already be working in
the kernel, I will report back later today.

-- Libo

> I notice that i2c_write has "[%*phD]" which means it worked, and also
> skipped out on the verification tests. This should be fixed.
> 
> But trace-cmd and perf will not be able to print that out either.
> 
> Now %*pbl works in the kernel (not checked), but probably doesn't work for
> trace-cmd.
> 
> -- Steve

On 3/28/25 15:28, Libo Chen wrote:
> 
> 
> On 3/27/25 07:03, Steven Rostedt wrote:
>> On Wed, 26 Mar 2025 21:32:32 -0700
>> Libo Chen <libo.chen@oracle.com> wrote:
>>
>>> On 3/26/25 19:06, Steven Rostedt wrote:
>>>> On Wed, 26 Mar 2025 17:40:02 -0700
>>>> Libo Chen <libo.chen@oracle.com> wrote:
>>>>   
>>>>>> +	TP_printk("comm=%s pid=%d tgid=%d ngid=%d mem_node_allowed_mask=%lx",    
>>>>>
>>>>>
>>>>> I cannot find a way to print out nodemask_t nicely here with %*pbl.
>>>>> So I fall back to just raw hex value. Will be grateful if someone
>>>>> knows a better way to print nodemask nicely in a tracepoint    
>>>>
>>>> Ug.
>>>>
>>>> I guess I need to add support for that. Both in the boot verifier as well
>>>> as in libtraceevent.
>>>>   
>>>
>>> That will be fantastic, I think this is quite useful.
>>
> 
> Oops just saw this right now, not sure how I missed it.
>  
>> I'm curious to what was broken. Just the user space side? Or was the kernel
>> side broken too?
>>
> 
> That's a good point, I originally only tried %*pbl with trace-cmd, but I
> tested your patch without trace-cmd. So yeah %*pbl may already be working in
> the kernel, I will report back later today.
> 
> -- Libo
> 

Okay yes, it does prints out to trace log. Not seeing the output is due to the
userspace tool not the lack of check. 

-- Libo

>> I notice that i2c_write has "[%*phD]" which means it worked, and also
>> skipped out on the verification tests. This should be fixed.
>>
>> But trace-cmd and perf will not be able to print that out either.
>>
>> Now %*pbl works in the kernel (not checked), but probably doesn't work for
>> trace-cmd.
>>
>> -- Steve
>