The NMI-source bitmap is the most critical information provided by the
NMI-source reporting feature. It can help identify issues when multiple
NMIs occur simultaneously or if certain NMI handlers consistently
misbehave. It is also very useful in debugging unknown NMIs since it can
pinpoint the exact source that caused the NMI.

Add the source bitmap to the nmi_handler() tracepoint. Also, print the
bitmap along with the "Unknown NMI" kernel log message.

Signed-off-by: Sohil Mehta <sohil.mehta@intel.com>
---
v5: New patch
---
 arch/x86/kernel/nmi.c      |  5 ++++-
 include/trace/events/nmi.h | 13 ++++++++-----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
index 183e3e717326..b9ece0b63ca7 100644
--- a/arch/x86/kernel/nmi.c
+++ b/arch/x86/kernel/nmi.c
@@ -202,7 +202,7 @@ static int nmi_handle(unsigned int type, struct pt_regs *regs)
 		thishandled = a->handler(type, regs);
 		handled += thishandled;
 		delta = sched_clock() - delta;
-		trace_nmi_handler(a->handler, (int)delta, thishandled);
+		trace_nmi_handler(a->handler, (int)delta, thishandled, source_bitmap);
 
 		nmi_check_duration(a, delta);
 	}
@@ -387,6 +387,9 @@ unknown_nmi_error(unsigned char reason, struct pt_regs *regs)
 	pr_emerg_ratelimited("Uhhuh. NMI received for unknown reason %02x on CPU %d.\n",
 			     reason, smp_processor_id());
 
+	if (cpu_feature_enabled(X86_FEATURE_NMI_SOURCE))
+		pr_emerg_ratelimited("NMI-source bitmap is 0x%lx\n", fred_event_data(regs));
+
 	if (unknown_nmi_panic || panic_on_unrecovered_nmi)
 		nmi_panic(regs, "NMI: Not continuing");
 
diff --git a/include/trace/events/nmi.h b/include/trace/events/nmi.h
index 18e0411398ba..6e4a1ff70a44 100644
--- a/include/trace/events/nmi.h
+++ b/include/trace/events/nmi.h
@@ -10,29 +10,32 @@
 
 TRACE_EVENT(nmi_handler,
 
-	TP_PROTO(void *handler, s64 delta_ns, int handled),
+	TP_PROTO(void *handler, s64 delta_ns, int handled, unsigned long source_bitmap),
 
-	TP_ARGS(handler, delta_ns, handled),
+	TP_ARGS(handler, delta_ns, handled, source_bitmap),
 
 	TP_STRUCT__entry(
 		__field(	void *,		handler	)
 		__field(	s64,		delta_ns)
 		__field(	int,		handled	)
+		__field(	unsigned long,	source_bitmap)
 	),
 
 	TP_fast_assign(
 		__entry->handler = handler;
 		__entry->delta_ns = delta_ns;
 		__entry->handled = handled;
+		__entry->source_bitmap = source_bitmap;
 	),
 
-	TP_printk("%ps() delta_ns: %lld handled: %d",
+	TP_printk("%ps() delta_ns: %lld handled: %d source_bitmap: 0x%lx",
 		__entry->handler,
 		__entry->delta_ns,
-		__entry->handled)
+		__entry->handled,
+		__entry->source_bitmap)
 );
 
 #endif /* _TRACE_NMI_H */
 
-/* This part ust be outside protection */
+/* This part must be outside protection */
 #include <trace/define_trace.h>
-- 
2.43.0

On Tue,  6 May 2025 18:21:45 -0700
Sohil Mehta <sohil.mehta@intel.com> wrote:

> diff --git a/include/trace/events/nmi.h b/include/trace/events/nmi.h
> index 18e0411398ba..6e4a1ff70a44 100644
> --- a/include/trace/events/nmi.h
> +++ b/include/trace/events/nmi.h
> @@ -10,29 +10,32 @@
>  
>  TRACE_EVENT(nmi_handler,
>  
> -	TP_PROTO(void *handler, s64 delta_ns, int handled),
> +	TP_PROTO(void *handler, s64 delta_ns, int handled, unsigned long source_bitmap),

Even though x86 is currently the only architecture using the nmi
tracepoint, this "source_bitmap" makes it become very x86 specific.

This file should be moved into arch/x86/include/asm/trace/

And that would require adding to the Makefile:

CFLAGS_nmi.o := -I $(src)/../include/asm/trace

>  
> -	TP_ARGS(handler, delta_ns, handled),
> +	TP_ARGS(handler, delta_ns, handled, source_bitmap),
>  
>  	TP_STRUCT__entry(
>  		__field(	void *,		handler	)
>  		__field(	s64,		delta_ns)
>  		__field(	int,		handled	)
> +		__field(	unsigned long,	source_bitmap)
>  	),
>  
>  	TP_fast_assign(
>  		__entry->handler = handler;
>  		__entry->delta_ns = delta_ns;
>  		__entry->handled = handled;
> +		__entry->source_bitmap = source_bitmap;
>  	),
>  
> -	TP_printk("%ps() delta_ns: %lld handled: %d",
> +	TP_printk("%ps() delta_ns: %lld handled: %d source_bitmap: 0x%lx",
>  		__entry->handler,
>  		__entry->delta_ns,
> -		__entry->handled)
> +		__entry->handled,
> +		__entry->source_bitmap)
>  );
>  
>  #endif /* _TRACE_NMI_H */
>  
> -/* This part ust be outside protection */
> +/* This part must be outside protection */

And this would need to have:

#undef TRACE_INCLUDE_PATH
#undef TRACE_INCLUDE_FILE
#define TRACE_INCLUDE_PATH .
#define TRACE_INCLUDE_FILE nmi

-- Steve

>  #include <trace/define_trace.h>

On 5/7/2025 2:48 PM, Steven Rostedt wrote:
> On Tue,  6 May 2025 18:21:45 -0700
> Sohil Mehta <sohil.mehta@intel.com> wrote:
> 
>> diff --git a/include/trace/events/nmi.h b/include/trace/events/nmi.h
>> index 18e0411398ba..6e4a1ff70a44 100644
>> --- a/include/trace/events/nmi.h
>> +++ b/include/trace/events/nmi.h
>> @@ -10,29 +10,32 @@
>>  
>>  TRACE_EVENT(nmi_handler,
>>  
>> -	TP_PROTO(void *handler, s64 delta_ns, int handled),
>> +	TP_PROTO(void *handler, s64 delta_ns, int handled, unsigned long source_bitmap),
> 
> Even though x86 is currently the only architecture using the nmi
> tracepoint, this "source_bitmap" makes it become very x86 specific.
> 

Sure. Will move it into x86.

> This file should be moved into arch/x86/include/asm/trace/
> 
> And that would require adding to the Makefile:
> 
> CFLAGS_nmi.o := -I $(src)/../include/asm/trace
> 

Thank you for the detailed instructions. It makes it a lot easier.


> diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
> index 183e3e717326..b9ece0b63ca7 100644
> --- a/arch/x86/kernel/nmi.c
> +++ b/arch/x86/kernel/nmi.c
> @@ -202,7 +202,7 @@ static int nmi_handle(unsigned int type, struct pt_regs *regs)
>  		thishandled = a->handler(type, regs);
>  		handled += thishandled;
>  		delta = sched_clock() - delta;
> -		trace_nmi_handler(a->handler, (int)delta, thishandled);
> +		trace_nmi_handler(a->handler, (int)delta, thishandled, source_bitmap);
>  
>  		nmi_check_duration(a, delta);
>  	}

Also, I just realized that the source_bitmap information might be
incorrect for !NMI_LOCAL. With the new changes suggested by PeterZ in
patch 5, the source_bitmap would be significantly different from the
original one received from fred_event_data().

I need to figure out a solution to print it accurately in all situations.

Sohil