arch/x86/events/intel/core.c | 33 ++++++++++++++++++++++++--------- arch/x86/events/intel/ds.c | 8 +++++++- arch/x86/events/perf_event.h | 3 +++ 3 files changed, 34 insertions(+), 10 deletions(-)
The following commit has been merged into the perf/urgent branch of tip:
Commit-ID: a56c03a397e2cd0c4cf8da96dcd6214f7d0e7d8c
Gitweb: https://git.kernel.org/tip/a56c03a397e2cd0c4cf8da96dcd6214f7d0e7d8c
Author: Dapeng Mi <dapeng1.mi@linux.intel.com>
AuthorDate: Thu, 13 Aug 2026 14:43:46 +08:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 10 Sep 2026 10:22:50 +02:00
perf/x86/intel: Prevent drain_pebs() reentry
The PEBS buffer is shared by all events on a CPU, so drain_pebs() must
not be reentered. If so, one instance may observe stale buffer state and
potentially access out-of-bound memory.
Most invocations happen in NMI context, which naturally prevents reentry.
However, drain_pebs() is also reachable from process context via
intel_pmu_drain_pebs_buffer().
In those paths, the PMU is often already disabled, but not guaranteed.
For example, __intel_pmu_pebs_disable() only disables the target counter,
so other active counters can still raise a PMI and interrupt an in-flight
drain_pebs(). Here is an example,
__perf_addr_filters_adjust()
perf_event_stop()
__perf_event_stop()
x86_pmu_stop() (event->pmu->stop)
intel_pmu_disable_event()
intel_pmu_pebs_disable()
__intel_pmu_pebs_disable()
intel_pmu_drain_large_pebs()
intel_pmu_drain_pebs_buffer()
Introduce __intel_pmu_quiesce() and __intel_pmu_resume() helpers and
use them in intel_pmu_drain_large_pebs() to disable the full PMU
around the intel_pmu_drain_pebs_buffer() call, preventing reentry.
Also add a warning in intel_pmu_drain_pebs_buffer() when the full PMU is
not disabled.
Fixes: b752ea0c28e3 ("perf/x86/intel/ds: Flush PEBS DS when changing PEBS_DATA_CFG")
Signed-off-by: Dapeng Mi <dapeng1.mi@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260813064346.335458-1-dapeng1.mi@linux.intel.com
---
arch/x86/events/intel/core.c | 33 ++++++++++++++++++++++++---------
arch/x86/events/intel/ds.c | 8 +++++++-
arch/x86/events/perf_event.h | 3 +++
3 files changed, 34 insertions(+), 10 deletions(-)
diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index cc13164..1ac2ca3 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -3125,6 +3125,27 @@ static void intel_pmu_del_event(struct perf_event *event)
this_cpu_ptr(&cpu_hw_events)->n_late_setup--;
}
+int __intel_pmu_quiesce(void)
+{
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+ int pmu_enabled = cpuc->enabled;
+
+ cpuc->enabled = 0;
+ if (pmu_enabled)
+ intel_pmu_disable_all();
+
+ return pmu_enabled;
+}
+
+void __intel_pmu_resume(int pmu_enabled)
+{
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+ cpuc->enabled = pmu_enabled;
+ if (pmu_enabled)
+ intel_pmu_enable_all(0);
+}
+
static int icl_set_topdown_event_period(struct perf_event *event)
{
struct hw_perf_event *hwc = &event->hw;
@@ -3316,16 +3337,13 @@ static void intel_pmu_read_event(struct perf_event *event)
if (event->hw.flags & (PERF_X86_EVENT_AUTO_RELOAD | PERF_X86_EVENT_TOPDOWN) ||
is_pebs_counter_event_group(event)) {
struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
- bool pmu_enabled = cpuc->enabled;
+ int pmu_enabled;
/* Only need to call update_topdown_event() once for group read. */
if (is_metric_event(event) && (cpuc->txn_flags & PERF_PMU_TXN_READ))
return;
- cpuc->enabled = 0;
- if (pmu_enabled)
- intel_pmu_disable_all();
-
+ pmu_enabled = __intel_pmu_quiesce();
/*
* If the PEBS counters snapshotting is enabled,
* the topdown event is available in PEBS records.
@@ -3334,10 +3352,7 @@ static void intel_pmu_read_event(struct perf_event *event)
static_call(intel_pmu_update_topdown_event)(event, NULL);
else
intel_pmu_drain_pebs_buffer();
-
- cpuc->enabled = pmu_enabled;
- if (pmu_enabled)
- intel_pmu_enable_all(0);
+ __intel_pmu_resume(pmu_enabled);
return;
}
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index d0bb767..b98029b 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -1242,8 +1242,11 @@ unlock:
void intel_pmu_drain_pebs_buffer(void)
{
+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
struct perf_sample_data data;
+ WARN_ON_ONCE(cpuc->enabled);
+
static_call(x86_pmu_drain_pebs)(NULL, &data);
}
@@ -1864,8 +1867,11 @@ static void intel_pmu_pebs_via_pt_enable(struct perf_event *event)
static inline void intel_pmu_drain_large_pebs(struct cpu_hw_events *cpuc)
{
if (cpuc->n_pebs == cpuc->n_large_pebs &&
- cpuc->n_pebs != cpuc->n_pebs_via_pt)
+ cpuc->n_pebs != cpuc->n_pebs_via_pt) {
+ int enabled = __intel_pmu_quiesce();
intel_pmu_drain_pebs_buffer();
+ __intel_pmu_resume(enabled);
+ }
}
static void __intel_pmu_pebs_enable(struct perf_event *event)
diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h
index 71ed5b2..4680cba 100644
--- a/arch/x86/events/perf_event.h
+++ b/arch/x86/events/perf_event.h
@@ -1638,6 +1638,9 @@ static __always_inline void __intel_pmu_lbr_disable(void)
wrmsrq(MSR_IA32_DEBUGCTLMSR, debugctl);
}
+extern int __intel_pmu_quiesce(void);
+extern void __intel_pmu_resume(int pmu_enabled);
+
int intel_pmu_save_and_restart(struct perf_event *event);
struct event_constraint *© 2016 - 2026 Red Hat, Inc.